Add artifactory storage backend

cmd/proxy/actions/storage.go

@@ -9,6 +9,7 @@ import (
 	"github.com/gomods/athens/pkg/config"
 	"github.com/gomods/athens/pkg/errors"
 	"github.com/gomods/athens/pkg/storage"
+	"github.com/gomods/athens/pkg/storage/artifactory"
 	"github.com/gomods/athens/pkg/storage/azureblob"
 	"github.com/gomods/athens/pkg/storage/external"
 	"github.com/gomods/athens/pkg/storage/fs"
@@ -67,6 +68,11 @@ func GetStorage(storageType string, storageConfig *config.Storage, timeout time.
 			return nil, errors.E(op, "Invalid External Storage Configuration")
 		}
 		return external.NewClient(storageConfig.External.URL, client), nil
+	case "artifactory":
+		if storageConfig.Artifactory == nil {
+			return nil, errors.E(op, "Invalid External Artifactory Configuration")
+		}
+		return artifactory.New(storageConfig.Artifactory, client)
 	default:
 		return nil, fmt.Errorf("storage type %s is unknown", storageType)
 	}

config.dev.toml

@@ -123,7 +123,7 @@ RobotsFile = "robots.txt"
 Timeout = 300
 
 # StorageType sets the type of storage backend the proxy will use.
-# Possible values are memory, disk, mongo, gcp, minio, s3, azureblob, external
+# Possible values are memory, disk, mongo, gcp, minio, s3, azureblob, external, artifactory
 # Defaults to memory
 # Env override: ATHENS_STORAGE_TYPE
 StorageType = "memory"
@@ -516,6 +516,31 @@ IndexType = "none"
         # Env override: ATHENS_EXTERNAL_STORAGE_URL
         URL = ""
 
+   [Storage.Artifactory]
+        # URL for Artifactory instance
+        # Env override: ATHENS_ARTIFACTORY_URL
+        URL = "https://artifactory.example.com:8083/"
+
+        # Name of repository in artifactory
+        # Env override: ATHENS_ARTIFACTORY_REPOSITORY
+        Repository = "release"
+
+        # Username to use with basic authentication to artifactory
+        # Env override: ATHENS_ARTIFACTORY_USERNAME
+        Username = ""
+
+        # Password to use with basic authentication to artifactory
+        # Env override: ATHENS_ARTIFACTORY_PASSWORD
+        Password = ""
+
+        # APIKey to use with key authentication to artifactory
+        # Env override: ATHENS_ARTIFACTORY_API_KEY
+        APIKey = ""
+
+        # AccessToken to use with token authentication to artifactory
+        # Env override: ATHENS_ARTIFACTORY_ACCESS_TOKEN
+        AccessToken = ""
+
 [Index]
     [Index.MySQL]
         # MySQL protocol

pkg/config/artifactory.go

@@ -0,0 +1,11 @@
+package config
+
+// ArtifactoryConfig specifies configuration for an artifactory storage
+type ArtifactoryConfig struct {
+	URL         string `validate:"required" envconfig:"ATHENS_ARTIFACTORY_URL"`
+	Repository  string `validate:"required" envconfig:"ATHENS_ARTIFACTORY_REPOSITORY"`
+	Username    string `envconfig:"ATHENS_ARTIFACTORY_USERNAME"`
+	Password    string `envconfig:"ATHENS_ARTIFACTORY_PASSWORD"`
+	APIKey      string `envconfig:"ATHENS_ARTIFACTORY_API_KEY"`
+	AccessToken string `envconfig:"ATHENS_ARTIFACTORY_ACCESS_TOKEN"`
+}

pkg/config/config.go

@@ -331,6 +331,8 @@ func validateStorage(validate *validator.Validate, storageType string, config *S
 		return validate.Struct(config.AzureBlob)
 	case "external":
 		return validate.Struct(config.External)
+	case "artifactory":
+		return validate.Struct(config.Artifactory)
 	default:
 		return fmt.Errorf("storage type %q is unknown", storageType)
 	}

pkg/config/storage.go

@@ -2,11 +2,12 @@ package config
 
 // Storage provides configs for various storage backends
 type Storage struct {
-	Disk      *DiskConfig
-	GCP       *GCPConfig
-	Minio     *MinioConfig
-	Mongo     *MongoConfig
-	S3        *S3Config
-	AzureBlob *AzureBlobConfig
-	External  *External
+	Disk        *DiskConfig
+	GCP         *GCPConfig
+	Minio       *MinioConfig
+	Mongo       *MongoConfig
+	S3          *S3Config
+	AzureBlob   *AzureBlobConfig
+	External    *External
+	Artifactory *ArtifactoryConfig
 }

pkg/storage/artifactory/client.go

@@ -0,0 +1,272 @@
+package artifactory
+
+import (
+	"archive/zip"
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"path"
+	"strings"
+
+	"github.com/gomods/athens/pkg/config"
+	"github.com/gomods/athens/pkg/errors"
+	"github.com/gomods/athens/pkg/storage"
+
+	"golang.org/x/mod/module"
+)
+
+type service struct {
+	baseURL *url.URL
+	repo    string
+	user    string
+	pass    string
+	c       *http.Client
+}
+
+// New returns an artifactory storage client
+func New(conf *config.ArtifactoryConfig, c *http.Client) (storage.Backend, error) {
+	const op errors.Op = "artifactory.New"
+	if c == nil {
+		c = &http.Client{}
+	}
+	u, err := url.Parse(conf.URL)
+	if err != nil {
+		return nil, errors.E(op, err)
+	}
+	var password string
+	switch {
+	case conf.Password != "":
+		password = conf.Password
+	case conf.APIKey != "":
+		password = conf.APIKey
+	case conf.AccessToken != "":
+		password = conf.AccessToken
+	}
+	return &service{
+		baseURL: u,
+		repo:    conf.Repository,
+		user:    conf.Username,
+		pass:    password,
+		c:       c,
+	}, nil
+}
+
+func (s *service) req(ctx context.Context, method string, u *url.URL, body io.Reader) (*http.Request, error) {
+	const op errors.Op = "artifactory.req"
+	req, err := http.NewRequest(method, s.baseURL.ResolveReference(u).String(), body)
+	if err != nil {
+		return nil, errors.E(op, err)
+	}
+	req.WithContext(ctx)
+	if s.pass != "" || s.user != "" {
+		req.SetBasicAuth(s.user, s.pass)
+	}
+	return req, nil
+}
+
+func (s *service) fileKey(mod, version, ext string) (string, error) {
+	const op errors.Op = "artifactory.fileKey"
+	var err error
+	mod, err = module.EscapePath(mod)
+	if err != nil {
+		return "", errors.E(op, err)
+	}
+	if version == "" {
+		return path.Join(s.repo, mod), nil
+	}
+	if ext == "" {
+		return path.Join(s.repo, mod, version), nil
+	}
+	return path.Join(s.repo, mod, version, ext), nil
+}
+
+func (s *service) getRequest(ctx context.Context, mod, version, ext string) (*http.Request, error) {
+	const op errors.Op = "artifactory.getRequest"
+	fileKey, err := s.fileKey(mod, version, ext)
+	if err != nil {
+		return nil, errors.E(op, err)
+	}
+	req, err := s.req(ctx, http.MethodGet, &url.URL{
+		Path: fileKey,
+	}, nil)
+	if err != nil {
+		return nil, errors.E(op, err)
+	}
+	return req, nil
+}
+
+func (s *service) do(req *http.Request) ([]byte, error) {
+	const op errors.Op = "artifactory.do"
+	resp, err := s.c.Do(req)
+	if err != nil {
+		return nil, errors.E(op, err)
+	}
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, errors.E(op, err)
+	}
+	if resp.StatusCode != 200 {
+		return nil, errors.E(op, fmt.Errorf("non 200 status code: %v - body: %s", resp.StatusCode, body), resp.StatusCode)
+	}
+	return body, nil
+}
+
+func (s *service) List(ctx context.Context, mod string) ([]string, error) {
+	const op errors.Op = "artifactory.List"
+	fileKey, err := s.fileKey(mod, "", "")
+	if err != nil {
+		return nil, errors.E(op, err)
+	}
+	req, err := s.req(ctx, http.MethodGet, &url.URL{
+		Path: path.Join("api", "storage", fileKey),
+	}, nil)
+	if err != nil {
+		return nil, errors.E(op, err)
+	}
+	req.Header.Add("Accept", "application/vnd.org.jfrog.artifactory.storage.FolderInfo+json")
+	resp, err := s.c.Do(req)
+	if err != nil {
+		return nil, errors.E(op, err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode == 404 {
+		return nil, nil
+	} else if resp.StatusCode != 200 {
+		body, _ := ioutil.ReadAll(resp.Body)
+		return nil, errors.E(op, fmt.Errorf("non 200 status code: %v - body: %s", resp.StatusCode, body), resp.StatusCode)
+	}
+	var folderResp struct {
+		Children []struct {
+			URI    string `json:"uri"`
+			Folder bool   `json:"folder"`
+		} `json:"children"`
+	}
+	if err := json.NewDecoder(resp.Body).Decode(&folderResp); err != nil {
+		return nil, errors.E(op, err)
+	}
+	var versions []string
+	for _, child := range folderResp.Children {
+		if !child.Folder {
+			continue
+		}
+		version := strings.Trim(child.URI, "/")
+		versions = append(versions, version)
+	}
+	return versions, nil
+}
+
+func (s *service) Info(ctx context.Context, mod, ver string) ([]byte, error) {
+	const op errors.Op = "artifactory.Info"
+	req, err := s.getRequest(ctx, mod, ver, "mod.info")
+	if err != nil {
+		return nil, errors.E(op, err)
+	}
+	modFile, err := s.do(req)
+	if err != nil {
+		return nil, errors.E(op, err)
+	}
+	return modFile, nil
+}
+
+func (s *service) GoMod(ctx context.Context, mod, ver string) ([]byte, error) {
+	const op errors.Op = "artifactory.GoMod"
+	req, err := s.getRequest(ctx, mod, ver, "mod.mod")
+	if err != nil {
+		return nil, errors.E(op, err)
+	}
+	modFile, err := s.do(req)
+	if err != nil {
+		return nil, errors.E(op, err)
+	}
+	return modFile, nil
+}
+
+func (s *service) Zip(ctx context.Context, mod, ver string) (io.ReadCloser, error) {
+	const op errors.Op = "artifactory.Zip"
+	req, err := s.getRequest(ctx, mod, ver, "mod.zip")
+	if err != nil {
+		return nil, errors.E(op, err)
+	}
+	modFile, err := s.do(req)
+	if err != nil {
+		return nil, errors.E(op, err)
+	}
+	return ioutil.NopCloser(bytes.NewReader(modFile)), nil
+}
+
+func (s *service) Save(ctx context.Context, mod, ver string, modFile []byte, modZip io.Reader, info []byte) error {
+	const op errors.Op = "artifactory.Save"
+	var err error
+	fileKey, err := s.fileKey(mod, ver, ".zip")
+	if err != nil {
+		return errors.E(op, err)
+	}
+	pr, pw := io.Pipe()
+	zw := zip.NewWriter(pw)
+	go func() {
+		err := upload(zw, modFile, info, modZip)
+		pw.CloseWithError(err)
+	}()
+	req, err := s.req(ctx, http.MethodPut, &url.URL{
+		Path: fileKey,
+	}, pr)
+	if err != nil {
+		return errors.E(op, err)
+	}
+	req.Header.Add("Content-Type", "application/zip")
+	req.Header.Add("X-Explode-Archive", "true")
+	req.Header.Add("X-Explode-Archive-Atomic", "true")
+	if _, err := s.do(req); err != nil {
+		return errors.E(op, err)
+	}
+	return nil
+}
+
+func (s *service) Delete(ctx context.Context, mod, ver string) error {
+	const op errors.Op = "artifactory.Delete"
+	fileKey, err := s.fileKey(mod, ver, "")
+	if err != nil {
+		return errors.E(op, err)
+	}
+	req, err := s.req(ctx, http.MethodDelete, &url.URL{
+		Path: fileKey,
+	}, nil)
+	if _, err := s.do(req); err != nil {
+		return errors.E(op, err)
+	}
+	return nil
+}
+
+func upload(zw *zip.Writer, mod, info []byte, zip io.Reader) error {
+	defer zw.Close()
+	infoW, err := zw.Create("mod.info")
+	if err != nil {
+		return fmt.Errorf("error creating info file: %v", err)
+	}
+	_, err = infoW.Write(info)
+	if err != nil {
+		return fmt.Errorf("error writing info file: %v", err)
+	}
+	modW, err := zw.Create("mod.mod")
+	if err != nil {
+		return fmt.Errorf("error creating mod file: %v", err)
+	}
+	_, err = modW.Write(mod)
+	if err != nil {
+		return fmt.Errorf("error writing mod file: %v", err)
+	}
+	zipW, err := zw.Create("mod.zip")
+	if err != nil {
+		return fmt.Errorf("error creating zip file: %v", err)
+	}
+	_, err = io.Copy(zipW, zip)
+	if err != nil {
+		return fmt.Errorf("error writing zip file: %v", err)
+	}
+	return nil
+}