gps/deduce.go
// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package gps
import (
"context"
"fmt"
"io"
"io/ioutil"
"net/http"
"net/url"
"os"
"path"
"path/filepath"
"regexp"
"runtime"
"strconv"
"strings"
"sync"
radix "github.com/armon/go-radix"
"github.com/pkg/errors"
)
var (
gitSchemes = []string{"https", "ssh", "git", "http"}
bzrSchemes = []string{"https", "bzr+ssh", "bzr", "http"}
hgSchemes = []string{"https", "ssh", "http"}
svnSchemes = []string{"https", "http", "svn", "svn+ssh"}
gopkginSchemes = []string{"https", "http"}
netrc []netrcLine
readNetrcOnce sync.Once
)
const gopkgUnstableSuffix = "-unstable"
func validateVCSScheme(scheme, typ string) bool {
// everything allows plain ssh
if scheme == "ssh" {
return true
}
var schemes []string
switch typ {
case "git":
schemes = gitSchemes
case "bzr":
schemes = bzrSchemes
case "hg":
schemes = hgSchemes
case "svn":
schemes = svnSchemes
default:
panic(fmt.Sprint("unsupported vcs type", scheme))
}
for _, valid := range schemes {
if scheme == valid {
return true
}
}
return false
}
// Regexes for the different known import path flavors
var (
// This regex allows some usernames that github currently disallows. They
// have allowed them in the past.
ghRegex = regexp.MustCompile(`^(?P<root>github\.com(/[A-Za-z0-9][-A-Za-z0-9]*/[A-Za-z0-9_.\-]+))((?:/[A-Za-z0-9_.\-]+)*)$`)
gpinNewRegex = regexp.MustCompile(`^(?P<root>gopkg\.in(?:(/[a-zA-Z0-9][-a-zA-Z0-9]+)?)(/[a-zA-Z][-.a-zA-Z0-9]*)\.((?:v0|v[1-9][0-9]*)(?:\.0|\.[1-9][0-9]*){0,2}(?:-unstable)?)(?:\.git)?)((?:/[a-zA-Z0-9][-.a-zA-Z0-9]*)*)$`)
//gpinOldRegex = regexp.MustCompile(`^(?P<root>gopkg\.in/(?:([a-z0-9][-a-z0-9]+)/)?((?:v0|v[1-9][0-9]*)(?:\.0|\.[1-9][0-9]*){0,2}(-unstable)?)/([a-zA-Z][-a-zA-Z0-9]*)(?:\.git)?)((?:/[a-zA-Z][-a-zA-Z0-9]*)*)$`)
bbRegex = regexp.MustCompile(`^(?P<root>bitbucket\.org(?P<bitname>/[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+))((?:/[A-Za-z0-9_.\-]+)*)$`)
//lpRegex = regexp.MustCompile(`^(?P<root>launchpad\.net/([A-Za-z0-9-._]+)(/[A-Za-z0-9-._]+)?)(/.+)?`)
lpRegex = regexp.MustCompile(`^(?P<root>launchpad\.net(/[A-Za-z0-9-._]+))((?:/[A-Za-z0-9_.\-]+)*)?$`)
//glpRegex = regexp.MustCompile(`^(?P<root>git\.launchpad\.net/([A-Za-z0-9_.\-]+)|~[A-Za-z0-9_.\-]+/(\+git|[A-Za-z0-9_.\-]+)/[A-Za-z0-9_.\-]+)$`)
glpRegex = regexp.MustCompile(`^(?P<root>git\.launchpad\.net(/[A-Za-z0-9_.\-]+))((?:/[A-Za-z0-9_.\-]+)*)$`)
//gcRegex = regexp.MustCompile(`^(?P<root>code\.google\.com/[pr]/(?P<project>[a-z0-9\-]+)(\.(?P<subrepo>[a-z0-9\-]+))?)(/[A-Za-z0-9_.\-]+)*$`)
jazzRegex = regexp.MustCompile(`^(?P<root>hub\.jazz\.net(/git/[a-z0-9]+/[A-Za-z0-9_.\-]+))((?:/[A-Za-z0-9_.\-]+)*)$`)
apacheRegex = regexp.MustCompile(`^(?P<root>git\.apache\.org(/[a-z0-9_.\-]+\.git))((?:/[A-Za-z0-9_.\-]+)*)$`)
vcsExtensionRegex = regexp.MustCompile(`^(?P<root>([a-z0-9.\-]+\.)+[a-z0-9.\-]+(:[0-9]+)?/[A-Za-z0-9_.\-/~]*?\.(?P<vcs>bzr|git|hg|svn))((?:/[A-Za-z0-9_.\-]+)*)$`)
)
// Other helper regexes
var (
scpSyntaxRe = regexp.MustCompile(`^([a-zA-Z0-9_]+)@([a-zA-Z0-9._-]+):(.*)$`)
pathvld = regexp.MustCompile(`^([A-Za-z0-9-]+)(\.[A-Za-z0-9-]+)+(/[A-Za-z0-9-_.~]+)*$`)
)
func pathDeducerTrie() *deducerTrie {
dxt := newDeducerTrie()
dxt.Insert("github.com/", githubDeducer{regexp: ghRegex})
dxt.Insert("gopkg.in/", gopkginDeducer{regexp: gpinNewRegex})
dxt.Insert("bitbucket.org/", bitbucketDeducer{regexp: bbRegex})
dxt.Insert("launchpad.net/", launchpadDeducer{regexp: lpRegex})
dxt.Insert("git.launchpad.net/", launchpadGitDeducer{regexp: glpRegex})
dxt.Insert("hub.jazz.net/", jazzDeducer{regexp: jazzRegex})
dxt.Insert("git.apache.org/", apacheDeducer{regexp: apacheRegex})
return dxt
}
type pathDeducer interface {
// deduceRoot takes an import path such as
// "github.com/some-user/some-package/some-subpackage"
// and returns the root folder to where the version control
// system exists. For example, the root folder where .git exists.
// So the return of the above string would be
// "github.com/some-user/some-package"
deduceRoot(string) (string, error)
deduceSource(string, *url.URL) (maybeSources, error)
}
type githubDeducer struct {
regexp *regexp.Regexp
}
func (m githubDeducer) deduceRoot(path string) (string, error) {
v := m.regexp.FindStringSubmatch(path)
if v == nil {
return "", fmt.Errorf("%s is not a valid path for a source on github.com", path)
}
return "github.com" + v[2], nil
}
func (m githubDeducer) deduceSource(path string, u *url.URL) (maybeSources, error) {
v := m.regexp.FindStringSubmatch(path)
if v == nil {
return nil, fmt.Errorf("%s is not a valid path for a source on github.com", path)
}
u.Host = "github.com"
u.Path = v[2]
if u.Scheme == "ssh" && u.User != nil && u.User.Username() != "git" {
return nil, fmt.Errorf("github ssh must be accessed via the 'git' user; %s was provided", u.User.Username())
} else if u.Scheme != "" {
if !validateVCSScheme(u.Scheme, "git") {
return nil, fmt.Errorf("%s is not a valid scheme for accessing a git repository", u.Scheme)
}
if u.Scheme == "ssh" {
u.User = url.User("git")
}
return maybeSources{maybeGitSource{url: u}}, nil
}
mb := make(maybeSources, len(gitSchemes))
for k, scheme := range gitSchemes {
u2 := *u
if scheme == "ssh" {
u2.User = url.User("git")
}
u2.Scheme = scheme
mb[k] = maybeGitSource{url: &u2}
}
return mb, nil
}
type bitbucketDeducer struct {
regexp *regexp.Regexp
}
func (m bitbucketDeducer) deduceRoot(path string) (string, error) {
v := m.regexp.FindStringSubmatch(path)
if v == nil {
return "", fmt.Errorf("%s is not a valid path for a source on bitbucket.org", path)
}
return "bitbucket.org" + v[2], nil
}
func (m bitbucketDeducer) deduceSource(path string, u *url.URL) (maybeSources, error) {
v := m.regexp.FindStringSubmatch(path)
if v == nil {
return nil, fmt.Errorf("%s is not a valid path for a source on bitbucket.org", path)
}
u.Host = "bitbucket.org"
u.Path = v[2]
// This isn't definitive, but it'll probably catch most
isgit := strings.HasSuffix(u.Path, ".git") || (u.User != nil && u.User.Username() == "git")
ishg := strings.HasSuffix(u.Path, ".hg") || (u.User != nil && u.User.Username() == "hg")
// TODO(sdboyer) resolve scm ambiguity if needed by querying bitbucket's REST API
if u.Scheme != "" {
validgit, validhg := validateVCSScheme(u.Scheme, "git"), validateVCSScheme(u.Scheme, "hg")
if isgit {
if !validgit {
// This is unreachable for now, as the git schemes are a
// superset of the hg schemes
return nil, fmt.Errorf("%s is not a valid scheme for accessing a git repository", u.Scheme)
}
return maybeSources{maybeGitSource{url: u}}, nil
} else if ishg {
if !validhg {
return nil, fmt.Errorf("%s is not a valid scheme for accessing an hg repository", u.Scheme)
}
return maybeSources{maybeHgSource{url: u}}, nil
} else if !validgit && !validhg {
return nil, fmt.Errorf("%s is not a valid scheme for accessing either a git or hg repository", u.Scheme)
}
// No other choice, make an option for both git and hg
return maybeSources{
maybeHgSource{url: u},
maybeGitSource{url: u},
}, nil
}
mb := make(maybeSources, 0)
// git is probably more common, even on bitbucket. however, bitbucket
// appears to fail _extremely_ slowly on git pings (ls-remote) when the
// underlying repository is actually an hg repository, so it's better
// to try hg first.
if !isgit {
for _, scheme := range hgSchemes {
u2 := *u
if scheme == "ssh" {
u2.User = url.User("hg")
}
u2.Scheme = scheme
mb = append(mb, maybeHgSource{url: &u2})
}
}
if !ishg {
for _, scheme := range gitSchemes {
u2 := *u
if scheme == "ssh" {
u2.User = url.User("git")
}
u2.Scheme = scheme
mb = append(mb, maybeGitSource{url: &u2})
}
}
return mb, nil
}
type gopkginDeducer struct {
regexp *regexp.Regexp
}
func (m gopkginDeducer) deduceRoot(p string) (string, error) {
v, err := m.parseAndValidatePath(p)
if err != nil {
return "", err
}
return v[1], nil
}
func (m gopkginDeducer) parseAndValidatePath(p string) ([]string, error) {
v := m.regexp.FindStringSubmatch(p)
if v == nil {
return nil, fmt.Errorf("%s is not a valid path for a source on gopkg.in", p)
}
// We duplicate some logic from the gopkg.in server in order to validate the
// import path string without having to make a network request
if strings.Contains(v[4], ".") {
return nil, fmt.Errorf("%s is not a valid import path; gopkg.in only allows major versions (%q instead of %q)",
p, v[4][:strings.Index(v[4], ".")], v[4])
}
return v, nil
}
func (m gopkginDeducer) deduceSource(p string, u *url.URL) (maybeSources, error) {
// Reuse root detection logic for initial validation
v, err := m.parseAndValidatePath(p)
if err != nil {
return nil, err
}
// Putting a scheme on gopkg.in would be really weird, disallow it
if u.Scheme != "" {
return nil, fmt.Errorf("specifying alternate schemes on gopkg.in imports is not permitted")
}
// gopkg.in is always backed by github
u.Host = "github.com"
if v[2] == "" {
elem := v[3][1:]
u.Path = path.Join("/go-"+elem, elem)
} else {
u.Path = path.Join(v[2], v[3])
}
unstable := false
majorStr := v[4]
if strings.HasSuffix(majorStr, gopkgUnstableSuffix) {
unstable = true
majorStr = strings.TrimSuffix(majorStr, gopkgUnstableSuffix)
}
major, err := strconv.ParseUint(majorStr[1:], 10, 64)
if err != nil {
// this should only be reachable if there's an error in the regex
return nil, fmt.Errorf("could not parse %q as a gopkg.in major version", majorStr[1:])
}
mb := make(maybeSources, len(gopkginSchemes))
for k, scheme := range gopkginSchemes {
u2 := *u
u2.Scheme = scheme
mb[k] = maybeGopkginSource{
opath: v[1],
url: &u2,
major: major,
unstable: unstable,
}
}
return mb, nil
}
type launchpadDeducer struct {
regexp *regexp.Regexp
}
func (m launchpadDeducer) deduceRoot(path string) (string, error) {
// TODO(sdboyer) lp handling is nasty - there's ambiguities which can only really
// be resolved with a metadata request. See https://github.com/golang/go/issues/11436
v := m.regexp.FindStringSubmatch(path)
if v == nil {
return "", fmt.Errorf("%s is not a valid path for a source on launchpad.net", path)
}
return "launchpad.net" + v[2], nil
}
func (m launchpadDeducer) deduceSource(path string, u *url.URL) (maybeSources, error) {
v := m.regexp.FindStringSubmatch(path)
if v == nil {
return nil, fmt.Errorf("%s is not a valid path for a source on launchpad.net", path)
}
u.Host = "launchpad.net"
u.Path = v[2]
if u.Scheme != "" {
if !validateVCSScheme(u.Scheme, "bzr") {
return nil, fmt.Errorf("%s is not a valid scheme for accessing a bzr repository", u.Scheme)
}
return maybeSources{maybeBzrSource{url: u}}, nil
}
mb := make(maybeSources, len(bzrSchemes))
for k, scheme := range bzrSchemes {
u2 := *u
u2.Scheme = scheme
mb[k] = maybeBzrSource{url: &u2}
}
return mb, nil
}
type launchpadGitDeducer struct {
regexp *regexp.Regexp
}
func (m launchpadGitDeducer) deduceRoot(path string) (string, error) {
// TODO(sdboyer) same ambiguity issues as with normal bzr lp
v := m.regexp.FindStringSubmatch(path)
if v == nil {
return "", fmt.Errorf("%s is not a valid path for a source on git.launchpad.net", path)
}
return "git.launchpad.net" + v[2], nil
}
func (m launchpadGitDeducer) deduceSource(path string, u *url.URL) (maybeSources, error) {
v := m.regexp.FindStringSubmatch(path)
if v == nil {
return nil, fmt.Errorf("%s is not a valid path for a source on git.launchpad.net", path)
}
u.Host = "git.launchpad.net"
u.Path = v[2]
if u.Scheme != "" {
if !validateVCSScheme(u.Scheme, "git") {
return nil, fmt.Errorf("%s is not a valid scheme for accessing a git repository", u.Scheme)
}
return maybeSources{maybeGitSource{url: u}}, nil
}
mb := make(maybeSources, len(gitSchemes))
for k, scheme := range gitSchemes {
u2 := *u
u2.Scheme = scheme
mb[k] = maybeGitSource{url: &u2}
}
return mb, nil
}
type jazzDeducer struct {
regexp *regexp.Regexp
}
func (m jazzDeducer) deduceRoot(path string) (string, error) {
v := m.regexp.FindStringSubmatch(path)
if v == nil {
return "", fmt.Errorf("%s is not a valid path for a source on hub.jazz.net", path)
}
return "hub.jazz.net" + v[2], nil
}
func (m jazzDeducer) deduceSource(path string, u *url.URL) (maybeSources, error) {
v := m.regexp.FindStringSubmatch(path)
if v == nil {
return nil, fmt.Errorf("%s is not a valid path for a source on hub.jazz.net", path)
}
u.Host = "hub.jazz.net"
u.Path = v[2]
switch u.Scheme {
case "":
u.Scheme = "https"
fallthrough
case "https":
return maybeSources{maybeGitSource{url: u}}, nil
default:
return nil, fmt.Errorf("IBM's jazz hub only supports https, %s is not allowed", u.String())
}
}
type apacheDeducer struct {
regexp *regexp.Regexp
}
func (m apacheDeducer) deduceRoot(path string) (string, error) {
v := m.regexp.FindStringSubmatch(path)
if v == nil {
return "", fmt.Errorf("%s is not a valid path for a source on git.apache.org", path)
}
return "git.apache.org" + v[2], nil
}
func (m apacheDeducer) deduceSource(path string, u *url.URL) (maybeSources, error) {
v := m.regexp.FindStringSubmatch(path)
if v == nil {
return nil, fmt.Errorf("%s is not a valid path for a source on git.apache.org", path)
}
u.Host = "git.apache.org"
u.Path = v[2]
if u.Scheme != "" {
if !validateVCSScheme(u.Scheme, "git") {
return nil, fmt.Errorf("%s is not a valid scheme for accessing a git repository", u.Scheme)
}
return maybeSources{maybeGitSource{url: u}}, nil
}
mb := make(maybeSources, len(gitSchemes))
for k, scheme := range gitSchemes {
u2 := *u
u2.Scheme = scheme
mb[k] = maybeGitSource{url: &u2}
}
return mb, nil
}
type vcsExtensionDeducer struct {
regexp *regexp.Regexp
}
func (m vcsExtensionDeducer) deduceRoot(path string) (string, error) {
v := m.regexp.FindStringSubmatch(path)
if v == nil {
return "", fmt.Errorf("%s contains no vcs extension hints for matching", path)
}
return v[1], nil
}
func (m vcsExtensionDeducer) deduceSource(path string, u *url.URL) (maybeSources, error) {
v := m.regexp.FindStringSubmatch(path)
if v == nil {
return nil, fmt.Errorf("%s contains no vcs extension hints for matching", path)
}
switch v[4] {
case "git", "hg", "bzr":
x := strings.SplitN(v[1], "/", 2)
// TODO(sdboyer) is this actually correct for bzr?
u.Host = x[0]
u.Path = "/" + x[1]
if u.Scheme != "" {
if !validateVCSScheme(u.Scheme, v[4]) {
return nil, fmt.Errorf("%s is not a valid scheme for accessing %s repositories (path %s)", u.Scheme, v[4], path)
}
switch v[4] {
case "git":
return maybeSources{maybeGitSource{url: u}}, nil
case "bzr":
return maybeSources{maybeBzrSource{url: u}}, nil
case "hg":
return maybeSources{maybeHgSource{url: u}}, nil
}
}
var schemes []string
var mb maybeSources
var f func(k int, u *url.URL)
switch v[4] {
case "git":
schemes = gitSchemes
f = func(k int, u *url.URL) {
mb[k] = maybeGitSource{url: u}
}
case "bzr":
schemes = bzrSchemes
f = func(k int, u *url.URL) {
mb[k] = maybeBzrSource{url: u}
}
case "hg":
schemes = hgSchemes
f = func(k int, u *url.URL) {
mb[k] = maybeHgSource{url: u}
}
}
mb = make(maybeSources, len(schemes))
for k, scheme := range schemes {
u2 := *u
u2.Scheme = scheme
f(k, &u2)
}
return mb, nil
default:
return nil, fmt.Errorf("unknown repository type: %q", v[4])
}
}
// A deducer takes an import path and inspects it to determine where the
// corresponding project root should be. It applies a number of matching
// techniques, eventually falling back to an HTTP request for go-get metadata if
// none of the explicit rules succeed.
//
// The only real implementation is deductionCoordinator. The interface is
// primarily intended for testing purposes.
type deducer interface {
deduceRootPath(ctx context.Context, path string) (pathDeduction, error)
}
type deductionCoordinator struct {
suprvsr *supervisor
mut sync.RWMutex
rootxt *radix.Tree
deducext *deducerTrie
}
func newDeductionCoordinator(superv *supervisor) *deductionCoordinator {
dc := &deductionCoordinator{
suprvsr: superv,
rootxt: radix.New(),
deducext: pathDeducerTrie(),
}
return dc
}
// deduceRootPath takes an import path and attempts to deduce various
// metadata about it - what type of source should handle it, and where its
// "root" is (for vcs repositories, the repository root).
//
// If no errors are encountered, the returned pathDeduction will contain both
// the root path and a list of maybeSources, which can be subsequently used to
// create a handler that will manage the particular source.
func (dc *deductionCoordinator) deduceRootPath(ctx context.Context, path string) (pathDeduction, error) {
if err := dc.suprvsr.ctx.Err(); err != nil {
return pathDeduction{}, err
}
// First, check the rootxt to see if there's a prefix match - if so, we
// can return that and move on.
dc.mut.RLock()
prefix, data, has := dc.rootxt.LongestPrefix(path)
dc.mut.RUnlock()
if has && isPathPrefixOrEqual(prefix, path) {
switch d := data.(type) {
case maybeSources:
return pathDeduction{root: prefix, mb: d}, nil
case *httpMetadataDeducer:
// Multiple calls have come in for a similar path shape during
// the window in which the HTTP request to retrieve go get
// metadata is in flight. Fold this request in with the existing
// one(s) by calling the deduction method, which will avoid
// duplication of work through a sync.Once.
return d.deduce(ctx, path)
}
panic(fmt.Sprintf("unexpected %T in deductionCoordinator.rootxt: %v", data, data))
}
// No match. Try known path deduction first.
pd, err := dc.deduceKnownPaths(path)
if err == nil {
// Deduction worked; store it in the rootxt, send on retchan and
// terminate.
// FIXME(sdboyer) deal with changing path vs. root. Probably needs
// to be predeclared and reused in the hmd returnFunc
dc.mut.Lock()
dc.rootxt.Insert(pd.root, pd.mb)
dc.mut.Unlock()
return pd, nil
}
if err != errNoKnownPathMatch {
return pathDeduction{}, err
}
// The err indicates no known path matched. It's still possible that
// retrieving go get metadata might do the trick.
hmd := &httpMetadataDeducer{
basePath: path,
suprvsr: dc.suprvsr,
// The vanity deducer will call this func with a completed
// pathDeduction if it succeeds in finding one. We process it
// back through the action channel to ensure serialized
// access to the rootxt map.
returnFunc: func(pd pathDeduction) {
dc.mut.Lock()
dc.rootxt.Insert(pd.root, pd.mb)
dc.mut.Unlock()
},
}
// Save the hmd in the rootxt so that calls checking on similar
// paths made while the request is in flight can be folded together.
dc.mut.Lock()
dc.rootxt.Insert(path, hmd)
dc.mut.Unlock()
// Trigger the HTTP-backed deduction process for this requestor.
return hmd.deduce(ctx, path)
}
// pathDeduction represents the results of a successful import path deduction -
// a root path, plus a maybeSource that can be used to attempt to connect to
// the source.
type pathDeduction struct {
root string
mb maybeSources
}
var errNoKnownPathMatch = errors.New("no known path match")
func (dc *deductionCoordinator) deduceKnownPaths(path string) (pathDeduction, error) {
u, path, err := normalizeURI(path)
if err != nil {
return pathDeduction{}, err
}
// First, try the root path-based matches
if _, mtch, has := dc.deducext.LongestPrefix(path); has {
root, err := mtch.deduceRoot(path)
if err != nil {
return pathDeduction{}, err
}
mb, err := mtch.deduceSource(path, u)
if err != nil {
return pathDeduction{}, err
}
return pathDeduction{
root: root,
mb: mb,
}, nil
}
// Next, try the vcs extension-based (infix) matcher
exm := vcsExtensionDeducer{regexp: vcsExtensionRegex}
if root, err := exm.deduceRoot(path); err == nil {
mb, err := exm.deduceSource(path, u)
if err != nil {
return pathDeduction{}, err
}
return pathDeduction{
root: root,
mb: mb,
}, nil
}
return pathDeduction{}, errNoKnownPathMatch
}
type httpMetadataDeducer struct {
once sync.Once
deduced pathDeduction
deduceErr error
basePath string
returnFunc func(pathDeduction)
suprvsr *supervisor
}
func (hmd *httpMetadataDeducer) deduce(ctx context.Context, path string) (pathDeduction, error) {
hmd.once.Do(func() {
opath := path
u, path, err := normalizeURI(path)
if err != nil {
err = errors.Wrapf(err, "unable to normalize URI")
hmd.deduceErr = err
return
}
pd := pathDeduction{}
// Make the HTTP call to attempt to retrieve go-get metadata
var root, vcs, reporoot string
err = hmd.suprvsr.do(ctx, path, ctHTTPMetadata, func(ctx context.Context) error {
root, vcs, reporoot, err = getMetadata(ctx, path, u.Scheme)
if err != nil {
err = errors.Wrapf(err, "unable to read metadata")
}
return err
})
if err != nil {
err = errors.Wrapf(err, "unable to deduce repository and source type for %q", opath)
hmd.deduceErr = err
return
}
pd.root = root
// If we got something back at all, then it supersedes the actual input for
// the real URL to hit
repoURL, err := url.Parse(reporoot)
if err != nil {
err = errors.Wrapf(err, "server returned bad URL in go-get metadata, reporoot=%q", reporoot)
hmd.deduceErr = err
return
}
// If the input path specified a scheme, then try to honor it.
if u.Scheme != "" && repoURL.Scheme != u.Scheme {
// If the input scheme was http, but the go-get metadata
// nevertheless indicated https should be used for the repo, then
// trust the metadata and use https.
//
// To err on the secure side, do NOT allow the same in the other
// direction (https -> http).
if u.Scheme != "http" || repoURL.Scheme != "https" {
hmd.deduceErr = errors.Errorf("scheme mismatch for %q: input asked for %q, but go-get metadata specified %q", path, u.Scheme, repoURL.Scheme)
return
}
}
switch vcs {
case "git":
pd.mb = maybeSources{maybeGitSource{url: repoURL}}
case "bzr":
pd.mb = maybeSources{maybeBzrSource{url: repoURL}}
case "hg":
pd.mb = maybeSources{maybeHgSource{url: repoURL}}
default:
hmd.deduceErr = errors.Errorf("unsupported vcs type %s in go-get metadata from %s", vcs, path)
return
}
hmd.deduced = pd
// All data is assigned for other goroutines that may be waiting. Now,
// send the pathDeduction back to the deductionCoordinator by calling
// the returnFunc. This will also remove the reference to this hmd in
// the coordinator's trie.
//
// When this call finishes, it is guaranteed the coordinator will have
// at least begun running the action to insert the path deduction, which
// means no other deduction request will be able to interleave and
// request the same path before the pathDeduction can be processed, but
// after this hmd has been dereferenced from the trie.
hmd.returnFunc(pd)
})
return hmd.deduced, hmd.deduceErr
}
// normalizeURI takes a path string - which can be a plain import path, or a
// proper URI, or something SCP-shaped - performs basic validity checks, and
// returns both a full URL and just the path portion.
func normalizeURI(p string) (*url.URL, string, error) {
var u *url.URL
var newpath string
if m := scpSyntaxRe.FindStringSubmatch(p); m != nil {
// Match SCP-like syntax and convert it to a URL.
// Eg, "git@github.com:user/repo" becomes
// "ssh://git@github.com/user/repo".
u = &url.URL{
Scheme: "ssh",
User: url.User(m[1]),
Host: m[2],
Path: "/" + m[3],
// TODO(sdboyer) This is what stdlib sets; grok why better
//RawPath: m[3],
}
} else {
var err error
u, err = url.Parse(p)
if err != nil {
return nil, "", errors.Errorf("%q is not a valid URI", p)
}
}
// If no scheme was passed, then the entire path will have been put into
// u.Path. Either way, construct the normalized path correctly.
if u.Host == "" {
newpath = p
} else {
newpath = path.Join(u.Host, u.Path)
}
return u, newpath, nil
}
// fetchMetadata fetches the remote metadata for path.
func fetchMetadata(ctx context.Context, path, scheme string) (rc io.ReadCloser, err error) {
if scheme == "http" {
rc, err = doFetchMetadata(ctx, "http", path)
return
}
rc, err = doFetchMetadata(ctx, "https", path)
if err == nil {
return
}
rc, err = doFetchMetadata(ctx, "http", path)
return
}
func doFetchMetadata(ctx context.Context, scheme, path string) (io.ReadCloser, error) {
url := fmt.Sprintf("%s://%s?go-get=1", scheme, path)
switch scheme {
case "https", "http":
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return nil, errors.Wrapf(err, "unable to build HTTP request for URL %q", url)
}
req = addAuthFromNetrc(url, req)
resp, err := http.DefaultClient.Do(req.WithContext(ctx))
if err != nil {
return nil, errors.Wrapf(err, "failed HTTP request to URL %q", url)
}
return resp.Body, nil
default:
return nil, errors.Errorf("unknown remote protocol scheme: %q", scheme)
}
}
// See https://github.com/golang/go/blob/master/src/cmd/go/internal/web2/web.go
// for implementation
// Temporary netrc reader until https://github.com/golang/go/issues/31334 is solved
type netrcLine struct {
machine string
login string
password string
}
func parseNetrc(data string) []netrcLine {
// See https://www.gnu.org/software/inetutils/manual/html_node/The-_002enetrc-file.html
// for documentation on the .netrc format.
var nrc []netrcLine
var l netrcLine
inMacro := false
for _, line := range strings.Split(data, "\n") {
if inMacro {
if line == "" {
inMacro = false
}
continue
}
f := strings.Fields(line)
i := 0
for ; i < len(f)-1; i += 2 {
// Reset at each "machine" token.
// “The auto-login process searches the .netrc file for a machine token
// that matches […]. Once a match is made, the subsequent .netrc tokens
// are processed, stopping when the end of file is reached or another
// machine or a default token is encountered.”
switch f[i] {
case "machine":
l = netrcLine{machine: f[i+1]}
case "login":
l.login = f[i+1]
case "password":
l.password = f[i+1]
case "macdef":
// “A macro is defined with the specified name; its contents begin with
// the next .netrc line and continue until a null line (consecutive
// new-line characters) is encountered.”
inMacro = true
}
if l.machine != "" && l.login != "" && l.password != "" {
nrc = append(nrc, l)
l = netrcLine{}
}
}
if i < len(f) && f[i] == "default" {
// “There can be only one default token, and it must be after all machine tokens.”
break
}
}
return nrc
}
func netrcPath() (string, error) {
if env := os.Getenv("NETRC"); env != "" {
return env, nil
}
dir := os.Getenv("HOME")
base := ".netrc"
if runtime.GOOS == "windows" {
base = "_netrc"
}
return filepath.Join(dir, base), nil
}
// readNetrc parses a user's netrc file, ignoring any errors that occur.
func readNetrc() {
path, err := netrcPath()
if err != nil {
return
}
data, err := ioutil.ReadFile(path)
if err != nil {
return
}
netrc = parseNetrc(string(data))
}
// addAuthFromNetrc uses basic authentication on go-get requests
// for private repositories.
func addAuthFromNetrc(rawurl string, req *http.Request) *http.Request {
readNetrcOnce.Do(readNetrc)
for _, m := range netrc {
u, err := url.Parse(rawurl)
if err != nil {
continue
}
if u.Host == m.machine {
req.SetBasicAuth(m.login, m.password)
break
}
}
return req
}
// getMetadata fetches and decodes remote metadata for path.
//
// scheme is optional. If it's http, only http will be attempted for fetching.
// Any other scheme (including none) will first try https, then fall back to
// http.
func getMetadata(ctx context.Context, path, scheme string) (string, string, string, error) {
rc, err := fetchMetadata(ctx, path, scheme)
if err != nil {
return "", "", "", errors.Wrapf(err, "unable to fetch raw metadata")
}
defer rc.Close()
imports, err := parseMetaGoImports(rc)
if err != nil {
return "", "", "", errors.Wrapf(err, "unable to parse go-import metadata")
}
match := -1
for i, im := range imports {
if !strings.HasPrefix(path, im.Prefix) {
continue
}
if match != -1 {
return "", "", "", errors.Errorf("multiple meta tags match import path %q", path)
}
match = i
}
if match == -1 {
return "", "", "", errors.Errorf("go-import metadata not found")
}
return imports[match].Prefix, imports[match].VCS, imports[match].RepoRoot, nil
}