cloudfoundry-incubator/stratos

View on GitHub
src/jetstream/plugins/monocular/artifacthub.go

Summary

Maintainability
C
1 day
Test Coverage
package monocular

import (
    "encoding/json"
    "errors"
    "fmt"
    "io"
    "io/ioutil"
    "net/http"
    "os"
    "path"
    "sort"
    "strings"
    "time"

    "github.com/cloudfoundry-incubator/stratos/src/jetstream/plugins/monocular/store"
    "github.com/labstack/echo/v4"
    yaml "gopkg.in/yaml.v2"
)

// Artifact Hub support

type artifactHubHandler func(c echo.Context, endpointID string) error

const (
    searchURL = "https://artifacthub.io/api/chartsvc/v1/charts/search"
    // How long to cache data from the ArtifactHub API
    hubCacheExpiry = 15 * time.Minute
)

// Structs for data coming back from the ArtifactHub API
type ahData struct {
    Data []ahChart `json:"data"`
}

type ahChart struct {
    ID          string `json:"id"`
    ArtifactHub struct {
        PackageURL string `json:"packageUrl"`
    } `json:"artifactHub"`
    Attributes struct {
        Repo        Repo   `json:"repo"`
        Description string `json:"description"`
    } `json:"attributes"`
    Relationships struct {
        LatestVersion struct {
            Data struct {
                Version    string `json:"version"`
                AppVersion string `json:"app_version"`
            } `json:"data"`
        } `json:"latestChartVersion"`
    } `json:"relationships"`
}

type ahVersion struct {
    Version   string                `json:"version"`
    CreatedAt int64                 `json:"created_at"`
    SemVer    store.SemanticVersion `json:"-"`
}

type ahInfo struct {
    Description       string            `json:"description"`
    Name              string            `json:"name"`
    CreatedAt         int64             `json:"created_at"`
    Version           string            `json:"version"`
    Digest            string            `json:"digest"`
    HomeURL           string            `json:"home_url"`
    AppVersion        string            `json:"app_version"`
    IconID            string            `json:"logo_image_id"`
    Repository        Repo              `json:"repository"`
    AvailableVersions []ahVersion       `json:"available_versions"`
    Readme            string            `json:"readme"`
    Links             []Repo            `json:"links"`
    Maintainers       []ChartMaintainer `json:"maintainers"`
}

type ahVersions []ahVersion

// Look to see if the request is for ArtifactHub - if it is, invoke the specified request handler
func (m *Monocular) handleArtifactRequest(c echo.Context, handler artifactHubHandler) (bool, error) {
    externalMonocularEndpoint, err := m.isExternalMonocularRequest(c)
    if err != nil {
        return true, echo.NewHTTPError(http.StatusBadRequest, err.Error())
    }

    // If this request is associated with an external monocular instance forward the request on to it
    if externalMonocularEndpoint != nil {
        return true, handler(c, externalMonocularEndpoint.GUID)
    }
    return false, nil
}

// Fetch all charts from ArtifactHub using the Monocular-compatible search API
// We cache the results of the search on disk for the configfured cache period
func (m *Monocular) fetchChartsFromArtifactHub(c echo.Context, endpointID string) error {
    cacheFolder := path.Join(m.CacheFolder, endpointID)
    indexFile := path.Join(cacheFolder, "hub_index.json")
    if ok := useCachedFile(indexFile); ok {
        // Just send the cached file
        return c.File(indexFile)
    }

    // Fetch index of charts usign the search API
    httpClient := m.portalProxy.GetHttpClient(true)
    resp, err := httpClient.Get(searchURL)
    if err != nil {
        return err
    }
    defer resp.Body.Close()

    if resp.StatusCode != http.StatusOK {
        return fmt.Errorf("Error retrieving Helm Chart list from ArtifactHub - %d:%s", resp.StatusCode, resp.Status)
    }

    // Parse the list into JSON so we can translate it to be compatible with Monocular ressponse
    results := ahData{}
    decoder := json.NewDecoder(resp.Body)
    if err := decoder.Decode(&results); err != nil {
        return fmt.Errorf("Error marshalling Helm Chart list from ArtifactHub - %d:%s", resp.StatusCode, resp.Status)
    }

    // Translate response into a Monocular response
    charts := make(APIListResponse, len(results.Data))
    for index, info := range results.Data {
        ids := strings.Split(info.ID, "/")
        name := ids[len(ids)-1]

        chart := APIResponse{
            ID:   info.ID,
            Type: "chart",
        }
        chart.Attributes = Chart{
            Description: info.Attributes.Description,
            Name:        name,
            Repo:        info.Attributes.Repo,
            Icon:        fmt.Sprintf("/v1/hub/assets/%s/%s/logo", info.ID, info.Relationships.LatestVersion.Data.Version),
        }
        chart.Relationships = make(RelMap)
        rel := Rel{
            Data: ChartVersion{
                Version:    info.Relationships.LatestVersion.Data.Version,
                AppVersion: info.Relationships.LatestVersion.Data.AppVersion,
            },
        }
        chart.Relationships["latestChartVersion"] = rel
        charts[index] = &chart
    }

    response := BodyAPIListResponse{
        Data: &charts,
    }

    // Cache this response on disk, so next time we use it again
    if err := m.ensureFolder(cacheFolder); err == nil {
        if json, err := json.Marshal(response); err == nil {
            ioutil.WriteFile(indexFile, json, 0644)
        }
    }

    return c.JSON(200, response)
}

// Get a specific Chart from ArtifactHub
// We cache metadata on disk in the artifactHubGetPackageInfo function
func (m *Monocular) artifactHubGetChart(c echo.Context, endpointID string) error {
    repo := c.Param("repo")
    chartName := c.Param("name")
    version := c.Param("version")

    info, err := m.artifactHubGetPackageInfo(endpointID, repo, chartName, version)
    if err != nil {
        return err
    }

    // Transform the data into a Monocular response
    response := BodyAPIResponse{
        Data: APIResponse{
            ID:            fmt.Sprintf("%s/%s", repo, chartName),
            Type:          "chart",
            Relationships: make(map[string]Rel),
        },
    }

    attrs := Chart{
        Name: chartName,
        Repo: Repo{
            Name: repo,
        },
        Home:        info.HomeURL,
        Description: info.Description,
        Icon:        fmt.Sprintf("/v1/hub/assets/%s/%s/logo", response.Data.ID, version),
        Sources:     filterSourceLinks(info.Links),
        Maintainers: info.Maintainers,
    }
    response.Data.Attributes = attrs

    // Fill in the latest chart version
    sorted := m.artifactHubGetChartVersionsFromInfo(info, repo, chartName)
    if len(sorted) > 0 {
        lver := sorted[0]
        latest := ChartVersion{
            Version: lver.Version,
        }
        response.Data.Relationships["latestChartVersion"] = Rel{
            Data: latest,
        }
    }

    return c.JSON(200, response)
}

// Get the metadata for a specific version of a chart
// We need to download the Chart archive and unpack it in order to get the chart URL and information
// about whether the Chart has a schema
func (m *Monocular) artifactHubGetChartVersion(c echo.Context, endpointID string) error {
    repo := c.Param("repo")
    chartName := c.Param("name")
    version := c.Param("version")

    // Get the package info - this allows us to determine the URL of the Helm Repository
    // which we need in order to get the Chart URL and download
    info, err := m.artifactHubGetPackageInfo(endpointID, repo, chartName, version)
    if err != nil {
        return err
    }

    // Download the chart and cache required files
    cacheFolder, err := m.artifactHubCacheChartFiles(endpointID, repo, info.Repository.URL, chartName, version, info.Digest)
    if err != nil {
        return err
    }

    chartURL, err := ioutil.ReadFile(path.Join(cacheFolder, "chart_url"))
    if err != nil {
        return err
    }

    urls := make([]string, 1)
    urls[0] = string(chartURL)

    // Transform the data into a Monocular response
    response := BodyAPIResponse{
        Data: APIResponse{
            ID:            fmt.Sprintf("%s/%s-%s", repo, chartName, version),
            Type:          "chartVersion",
            Relationships: make(map[string]Rel),
        },
    }

    attrs := ChartVersion{
        AppVersion: info.AppVersion,
        Version:    info.Version,
        Created:    time.Unix(info.CreatedAt, 0),
        Digest:     info.Digest,
        Readme:     ahGetFileAssetURL(endpointID, repo, chartName, version, cacheFolder, "README.md"),
        Values:     ahGetFileAssetURL(endpointID, repo, chartName, version, cacheFolder, "values.yaml"),
        Schema:     ahGetFileAssetURL(endpointID, repo, chartName, version, cacheFolder, "values.schema.json"),
        URLs:       urls,
    }
    response.Data.Attributes = attrs

    chart := Chart{
        Icon:        fmt.Sprintf("/v1/hub/assets/%s/%s/%s/logo", repo, chartName, version),
        Maintainers: info.Maintainers,
    }
    response.Data.Relationships["chart"] = Rel{
        Data: chart,
    }

    return c.JSON(200, response)
}

// Return an asset URL if teh asset is available in the cache
func ahGetFileAssetURL(endpointID, repo, name, version, folder, filename string) string {
    cachePath := path.Join(folder, filename)
    if _, err := os.Stat(cachePath); os.IsNotExist(err) {
        return ""
    }
    return fmt.Sprintf("/v1/hub/%s/%s/%s/%s/%s", endpointID, repo, name, version, filename)
}

// Get a file for the given chart (readme, valuees, schema)
func (m *Monocular) artifactHubGetChartFile(c echo.Context) error {
    file := c.Param("file")
    return m.artifactHubGetChartFileNamed(c, file)
}

// Same as abuve, but allow name to be passed in, so we can use this internally too
func (m *Monocular) artifactHubGetChartFileNamed(c echo.Context, file string) error {
    endpointID := c.Param("endpoint")
    repo := c.Param("repo")
    chartName := c.Param("name")
    version := c.Param("version")

    if !isPermittedFile(file) {
        return echo.NewHTTPError(http.StatusNotFound, fmt.Sprintf("Can not find file %s for the specified chart", file))
    }

    info, err := m.artifactHubGetPackageInfo(endpointID, repo, chartName, version)
    if err != nil {
        return err
    }

    cacheFolder, err := m.artifactHubCacheChartFiles(endpointID, repo, info.Repository.URL, chartName, version, info.Digest)
    if err != nil {
        return err
    }

    fp := path.Join(cacheFolder, file)
    return c.File(fp)
}

// Get available versions for a Chart
func (m *Monocular) artifactHubGetChartVersions(c echo.Context, endpointID, repo, chartName string) ([]*store.ChartStoreRecord, error) {
    var versions store.ChartStoreRecordList
    info, err := m.artifactHubGetPackageInfo(endpointID, repo, chartName, "")
    if err != nil {
        return versions, err
    }
    return m.artifactHubGetChartVersionsFromInfo(info, repo, chartName), nil
}

// Get available versions for a Chart from ArtifactHub info
func (m *Monocular) artifactHubGetChartVersionsFromInfo(info *ahInfo, repo, chartName string) []*store.ChartStoreRecord {
    var versions store.ChartStoreRecordList
    // Translate to the bear miniumum version metadata
    for _, ver := range info.AvailableVersions {
        versions = append(versions, &store.ChartStoreRecord{
            Version:    ver.Version,
            Created:    time.Unix(ver.CreatedAt, 0),
            Name:       chartName,
            Repository: repo,
            SemVer:     store.NewSemanticVersion(ver.Version),
        })
    }
    sort.Sort(versions)
    return versions
}

// Get the icon for a Chart
func (m *Monocular) artifactHubGetIconHandler(c echo.Context, endpointID string) error {
    return m.artifactHubGetIcon(c)
}

func (m *Monocular) artifactHubGetIcon(c echo.Context) error {
    endpoint := c.Param("guid")
    repo := c.Param("repo")
    chartName := c.Param("name")
    version := c.Param("version")

    var contentType string

    // Look to see if we have the icon cached - fetch it if not
    iconFilePath := path.Join(m.CacheFolder, endpoint, fmt.Sprintf("%s_%s_%s", repo, chartName, version), "icon")
    iconTypeFilePath := path.Join(m.CacheFolder, endpoint, fmt.Sprintf("%s_%s_%s", repo, chartName, version), "icon.type")
    stats, err := os.Stat(iconFilePath)
    if os.IsNotExist(err) {
        // Not cached, so need to get chart info from ArtifactHub, cache icon and send
        hubInfo, err := m.artifactHubGetPackageInfo(endpoint, repo, chartName, version)
        if err != nil {
            return sendPlaceHolderIcon(c)
        }

        if err := m.ensureFolder(path.Dir(iconFilePath)); err != nil {
            return sendPlaceHolderIcon(c)
        }

        // No icon, so write a 0 byte file so next time we don't try and figure out it doesn't have one all over again
        if len(hubInfo.IconID) == 0 {
            out, err := os.Create(iconFilePath)
            if err == nil {
                out.Close()
            }
            return sendPlaceHolderIcon(c)
        }

        // Now download the icon
        iconURL := fmt.Sprintf("https://artifacthub.io/image/%s", hubInfo.IconID)
        contentType, err = m.downloadFile(iconFilePath, iconURL)
        if err != nil {
            return sendPlaceHolderIcon(c)
        }
        stats, err = os.Stat(iconFilePath)
        if err != nil {
            return sendPlaceHolderIcon(c)
        }

        // Write out the content type
        ioutil.WriteFile(iconTypeFilePath, []byte(contentType), 0644)
    }

    // If the file is 0 length
    if stats.Size() == 0 {
        return sendPlaceHolderIcon(c)
    }

    // Read the content type
    if len(contentType) == 0 {
        if data, err := ioutil.ReadFile(iconTypeFilePath); err == nil {
            contentType = string(data)
        }
    }

    iconFile, err := ioutil.ReadFile(iconFilePath)
    if err != nil {
        return sendPlaceHolderIcon(c)
    }
    c.Response().Header().Set("Content-Type", contentType)
    c.Response().Status = 200
    c.Response().Write(iconFile)

    return nil
}

func sendPlaceHolderIcon(c echo.Context) error {
    http.Redirect(c.Response().Writer, c.Request(), "/core/assets/custom/placeholder.png", http.StatusTemporaryRedirect)
    return nil
}

// Download package infor from ArtifactHub or use the cached version
func (m *Monocular) artifactHubGetPackageInfo(endpointID, repo, name, version string) (*ahInfo, error) {
    // Make sure we handle an empty version correctly
    var cacheName string
    var versionPart = ""
    if len(version) > 0 {
        versionPart = fmt.Sprintf("/%s", version)
        cacheName = fmt.Sprintf("%s_%s_%s.json", repo, name, version)
    } else {
        cacheName = fmt.Sprintf("%s_%s.json", repo, name)
    }

    var reader io.Reader
    fetch := true

    // Check for cached file
    cacheFolder := path.Join(m.CacheFolder, endpointID)
    indexFile := path.Join(cacheFolder, cacheName)

    if ok := useCachedFile(indexFile); ok {
        // Just use the cached file
        f, err := os.Open(indexFile)
        if err == nil {
            defer f.Close()
            reader = f
            fetch = false
        }
    }

    if fetch {
        infoURL := fmt.Sprintf("https://artifacthub.io/api/v1/packages/helm/%s/%s%s", repo, name, versionPart)
        httpClient := m.portalProxy.GetHttpClient(true)
        resp, err := httpClient.Get(infoURL)
        if err != nil {
            return nil, err
        }
        defer resp.Body.Close()

        if resp.StatusCode != http.StatusOK {
            return nil, fmt.Errorf("Error retrieving Helm Chart info from ArtifactHub - %d:%s", resp.StatusCode, resp.Status)
        }
        reader = resp.Body
    }

    result := ahInfo{}
    decoder := json.NewDecoder(reader)
    if err := decoder.Decode(&result); err != nil {
        return nil, fmt.Errorf("Error marshalling Helm Chart info from ArtifactHub - %+v", err)
    }

    // Cache the data if we fetched it
    if fetch {
        if err := m.ensureFolder(cacheFolder); err == nil {
            if json, err := json.Marshal(result); err == nil {
                ioutil.WriteFile(indexFile, json, 0644)
            }
        }
    }
    return &result, nil
}

// Cache the chart files if needed - in the same way we do for our built-in Monocular cache
func (m *Monocular) artifactHubCacheChartFiles(endpointID, repoName, repoURL, name, version, digest string) (string, error) {

    // First look to see if there is a digest file
    cacheFolder := path.Join(m.CacheFolder, endpointID, fmt.Sprintf("%s_%s_%s", repoName, name, version))
    if hasDigestFile(cacheFolder, digest) {
        return cacheFolder, nil
    }

    if err := m.ensureFolder(cacheFolder); err != nil {
        return cacheFolder, err
    }

    // We need to download the Helm Repository index file from the repo URL then look for our chart and version in order to find the Chart
    chartURL, err := m.getChartURL(repoURL, name, version)
    if err != nil {
        return cacheFolder, err
    }

    err = m.cacheChartFromURL(cacheFolder, digest, name, chartURL)
    if err != nil {
        return cacheFolder, err
    }

    // Write the chart URL to a file as well, so we don't have to do this again
    ioutil.WriteFile(path.Join(cacheFolder, "chart_url"), []byte(chartURL), 0644)
    return cacheFolder, nil
}

func useCachedFile(file string) bool {
    stats, err := os.Stat(file)
    if err == nil {
        // File exists - check how old it is
        expiryTime := stats.ModTime().Add(hubCacheExpiry)
        if expiryTime.Before(time.Now()) {
            // Delete the file
            os.Remove(file)
        } else {
            return true
        }
    }
    return false
}

func filterSourceLinks(links []Repo) []string {
    var sources []string
    for _, link := range links {
        if link.Name == "source" {
            sources = append(sources, link.URL)
        }
    }
    return sources
}

// Download the Helm Repository index and look for the specified chart and version and return the download URL for the chart
func (m *Monocular) getChartURL(repoURL, name, version string) (string, error) {
    httpClient := m.portalProxy.GetHttpClient(true)

    helmIndexURL := joinURL(repoURL, "index.yaml")
    resp, err := httpClient.Get(helmIndexURL)

    if err != nil {
        return "", fmt.Errorf("Could not download Helm Repository Index: %s", err)
    }
    if resp.StatusCode != 200 {
        return "", fmt.Errorf("Could not download Helm Repository Index: %s", resp.Status)
    }

    defer resp.Body.Close()

    // Marshal to the index structure
    var index IndexFile
    decoder := yaml.NewDecoder(resp.Body)
    err = decoder.Decode(&index)
    if err != nil {
        return "", fmt.Errorf("Error marshalling Helm Repository Index: %+v", err)
    }

    // Find the required version
    if chart, ok := index.Entries[name]; ok {
        for _, v := range chart {
            if v.Version == version {
                if len(v.URLs) > 0 {
                    chartURL := v.URLs[0]
                    return makeAbsoluteChartURL(chartURL, repoURL), nil
                }
            }
        }
    }

    return "", errors.New("Can not find Chart in the Helm Repository")
}