api/git/service.go
package git
import (
"context"
"strconv"
"strings"
"sync"
"time"
lru "github.com/hashicorp/golang-lru"
"github.com/rs/zerolog/log"
)
const (
repositoryCacheSize = 4
repositoryCacheTTL = 5 * time.Minute
)
// baseOption provides a minimum group of information to operate a git repository, like git-remote
type baseOption struct {
repositoryUrl string
username string
password string
tlsSkipVerify bool
}
// fetchOption allows to specify the reference name of the target repository
type fetchOption struct {
baseOption
referenceName string
dirOnly bool
}
// cloneOption allows to add a history truncated to the specified number of commits
type cloneOption struct {
fetchOption
depth int
}
type repoManager interface {
download(ctx context.Context, dst string, opt cloneOption) error
latestCommitID(ctx context.Context, opt fetchOption) (string, error)
listRefs(ctx context.Context, opt baseOption) ([]string, error)
listFiles(ctx context.Context, opt fetchOption) ([]string, error)
}
// Service represents a service for managing Git.
type Service struct {
shutdownCtx context.Context
azure repoManager
git repoManager
timerStopped bool
mut sync.Mutex
cacheEnabled bool
// Cache the result of repository refs, key is repository URL
repoRefCache *lru.Cache
// Cache the result of repository file tree, key is the concatenated string of repository URL and ref value
repoFileCache *lru.Cache
}
// NewService initializes a new service.
func NewService(ctx context.Context) *Service {
return newService(ctx, repositoryCacheSize, repositoryCacheTTL)
}
func newService(ctx context.Context, cacheSize int, cacheTTL time.Duration) *Service {
service := &Service{
shutdownCtx: ctx,
azure: NewAzureClient(),
git: NewGitClient(false),
timerStopped: false,
cacheEnabled: cacheSize > 0,
}
if service.cacheEnabled {
var err error
service.repoRefCache, err = lru.New(cacheSize)
if err != nil {
log.Debug().Err(err).Msg("failed to create ref cache")
}
service.repoFileCache, err = lru.New(cacheSize)
if err != nil {
log.Debug().Err(err).Msg("failed to create file cache")
}
if cacheTTL > 0 {
go service.startCacheCleanTimer(cacheTTL)
}
}
return service
}
// startCacheCleanTimer starts a timer to purge caches periodically
func (service *Service) startCacheCleanTimer(d time.Duration) {
ticker := time.NewTicker(d)
for {
select {
case <-ticker.C:
service.purgeCache()
case <-service.shutdownCtx.Done():
ticker.Stop()
service.mut.Lock()
service.timerStopped = true
service.mut.Unlock()
return
}
}
}
// timerHasStopped shows the CacheClean timer state with thread-safe way
func (service *Service) timerHasStopped() bool {
service.mut.Lock()
defer service.mut.Unlock()
ret := service.timerStopped
return ret
}
// CloneRepository clones a git repository using the specified URL in the specified
// destination folder.
func (service *Service) CloneRepository(destination, repositoryURL, referenceName, username, password string, tlsSkipVerify bool) error {
options := cloneOption{
fetchOption: fetchOption{
baseOption: baseOption{
repositoryUrl: repositoryURL,
username: username,
password: password,
tlsSkipVerify: tlsSkipVerify,
},
referenceName: referenceName,
},
depth: 1,
}
return service.cloneRepository(destination, options)
}
func (service *Service) cloneRepository(destination string, options cloneOption) error {
if isAzureUrl(options.repositoryUrl) {
return service.azure.download(context.TODO(), destination, options)
}
return service.git.download(context.TODO(), destination, options)
}
// LatestCommitID returns SHA1 of the latest commit of the specified reference
func (service *Service) LatestCommitID(repositoryURL, referenceName, username, password string, tlsSkipVerify bool) (string, error) {
options := fetchOption{
baseOption: baseOption{
repositoryUrl: repositoryURL,
username: username,
password: password,
tlsSkipVerify: tlsSkipVerify,
},
referenceName: referenceName,
}
if isAzureUrl(options.repositoryUrl) {
return service.azure.latestCommitID(context.TODO(), options)
}
return service.git.latestCommitID(context.TODO(), options)
}
// ListRefs will list target repository's references without cloning the repository
func (service *Service) ListRefs(repositoryURL, username, password string, hardRefresh bool, tlsSkipVerify bool) ([]string, error) {
refCacheKey := generateCacheKey(repositoryURL, username, password, strconv.FormatBool(tlsSkipVerify))
if service.cacheEnabled && hardRefresh {
// Should remove the cache explicitly, so that the following normal list can show the correct result
service.repoRefCache.Remove(refCacheKey)
// Remove file caches pointed to the same repository
for _, fileCacheKey := range service.repoFileCache.Keys() {
key, ok := fileCacheKey.(string)
if ok {
if strings.HasPrefix(key, repositoryURL) {
service.repoFileCache.Remove(key)
}
}
}
}
if service.repoRefCache != nil {
// Lookup the refs cache first
cache, ok := service.repoRefCache.Get(refCacheKey)
if ok {
refs, success := cache.([]string)
if success {
return refs, nil
}
}
}
options := baseOption{
repositoryUrl: repositoryURL,
username: username,
password: password,
tlsSkipVerify: tlsSkipVerify,
}
var (
refs []string
err error
)
if isAzureUrl(options.repositoryUrl) {
refs, err = service.azure.listRefs(context.TODO(), options)
if err != nil {
return nil, err
}
} else {
refs, err = service.git.listRefs(context.TODO(), options)
if err != nil {
return nil, err
}
}
if service.cacheEnabled && service.repoRefCache != nil {
service.repoRefCache.Add(refCacheKey, refs)
}
return refs, nil
}
// ListFiles will list all the files of the target repository with specific extensions.
// If extension is not provided, it will list all the files under the target repository
func (service *Service) ListFiles(repositoryURL, referenceName, username, password string, dirOnly, hardRefresh bool, includedExts []string, tlsSkipVerify bool) ([]string, error) {
repoKey := generateCacheKey(repositoryURL, referenceName, username, password, strconv.FormatBool(tlsSkipVerify), strconv.FormatBool(dirOnly))
if service.cacheEnabled && hardRefresh {
// Should remove the cache explicitly, so that the following normal list can show the correct result
service.repoFileCache.Remove(repoKey)
}
if service.repoFileCache != nil {
// lookup the files cache first
cache, ok := service.repoFileCache.Get(repoKey)
if ok {
files, success := cache.([]string)
if success {
// For the case while searching files in a repository without include extensions for the first time,
// but with include extensions for the second time
includedFiles := filterFiles(files, includedExts)
return includedFiles, nil
}
}
}
options := fetchOption{
baseOption: baseOption{
repositoryUrl: repositoryURL,
username: username,
password: password,
tlsSkipVerify: tlsSkipVerify,
},
referenceName: referenceName,
dirOnly: dirOnly,
}
var (
files []string
err error
)
if isAzureUrl(options.repositoryUrl) {
files, err = service.azure.listFiles(context.TODO(), options)
if err != nil {
return nil, err
}
} else {
files, err = service.git.listFiles(context.TODO(), options)
if err != nil {
return nil, err
}
}
includedFiles := filterFiles(files, includedExts)
if service.cacheEnabled && service.repoFileCache != nil {
service.repoFileCache.Add(repoKey, includedFiles)
return includedFiles, nil
}
return includedFiles, nil
}
func (service *Service) purgeCache() {
if service.repoRefCache != nil {
service.repoRefCache.Purge()
}
if service.repoFileCache != nil {
service.repoFileCache.Purge()
}
}
func generateCacheKey(names ...string) string {
return strings.Join(names, "-")
}
func matchExtensions(target string, exts []string) bool {
if len(exts) == 0 {
return true
}
for _, ext := range exts {
if strings.HasSuffix(target, ext) {
return true
}
}
return false
}
func filterFiles(paths []string, includedExts []string) []string {
if len(includedExts) == 0 {
return paths
}
var includedFiles []string
for _, filename := range paths {
// filter out the filenames with non-included extension
if matchExtensions(filename, includedExts) {
includedFiles = append(includedFiles, filename)
}
}
return includedFiles
}