Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add artifactory storage backend #1671

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions cmd/proxy/actions/storage.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"github.com/gomods/athens/pkg/config"
"github.com/gomods/athens/pkg/errors"
"github.com/gomods/athens/pkg/storage"
"github.com/gomods/athens/pkg/storage/artifactory"
"github.com/gomods/athens/pkg/storage/azureblob"
"github.com/gomods/athens/pkg/storage/external"
"github.com/gomods/athens/pkg/storage/fs"
Expand Down Expand Up @@ -67,6 +68,11 @@ func GetStorage(storageType string, storageConfig *config.Storage, timeout time.
return nil, errors.E(op, "Invalid External Storage Configuration")
}
return external.NewClient(storageConfig.External.URL, client), nil
case "artifactory":
if storageConfig.Artifactory == nil {
return nil, errors.E(op, "Invalid External Artifactory Configuration")
}
return artifactory.New(storageConfig.Artifactory, client)
default:
return nil, fmt.Errorf("storage type %s is unknown", storageType)
}
Expand Down
27 changes: 26 additions & 1 deletion config.dev.toml
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@ RobotsFile = "robots.txt"
Timeout = 300

# StorageType sets the type of storage backend the proxy will use.
# Possible values are memory, disk, mongo, gcp, minio, s3, azureblob, external
# Possible values are memory, disk, mongo, gcp, minio, s3, azureblob, external, artifactory
# Defaults to memory
# Env override: ATHENS_STORAGE_TYPE
StorageType = "memory"
Expand Down Expand Up @@ -499,6 +499,31 @@ IndexType = "none"
# Env override: ATHENS_EXTERNAL_STORAGE_URL
URL = ""

[Storage.Artifactory]
# URL for Artifactory instance
# Env override: ATHENS_ARTIFACTORY_URL
URL = "https://artifactory.example.com:8083/"

# Name of repository in artifactory
# Env override: ATHENS_ARTIFACTORY_REPOSITORY
Repository = "release"

# Username to use with basic authentication to artifactory
# Env override: ATHENS_ARTIFACTORY_USERNAME
Username = ""

# Password to use with basic authentication to artifactory
# Env override: ATHENS_ARTIFACTORY_PASSWORD
Password = ""

# APIKey to use with key authentication to artifactory
# Env override: ATHENS_ARTIFACTORY_API_KEY
APIKey = ""

# AccessToken to use with token authentication to artifactory
# Env override: ATHENS_ARTIFACTORY_ACCESS_TOKEN
AccessToken = ""

[Index]
[Index.MySQL]
# MySQL protocol
Expand Down
11 changes: 11 additions & 0 deletions pkg/config/artifactory.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
package config

// ArtifactoryConfig specifies configuration for an artifactory storage
type ArtifactoryConfig struct {
URL string `validate:"required" envconfig:"ATHENS_ARTIFACTORY_URL"`
Repository string `validate:"required" envconfig:"ATHENS_ARTIFACTORY_REPOSITORY"`
Username string `envconfig:"ATHENS_ARTIFACTORY_USERNAME"`
Password string `envconfig:"ATHENS_ARTIFACTORY_PASSWORD"`
APIKey string `envconfig:"ATHENS_ARTIFACTORY_API_KEY"`
AccessToken string `envconfig:"ATHENS_ARTIFACTORY_ACCESS_TOKEN"`
}
2 changes: 2 additions & 0 deletions pkg/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -329,6 +329,8 @@ func validateStorage(validate *validator.Validate, storageType string, config *S
return validate.Struct(config.AzureBlob)
case "external":
return validate.Struct(config.External)
case "artifactory":
return validate.Struct(config.Artifactory)
default:
return fmt.Errorf("storage type %q is unknown", storageType)
}
Expand Down
15 changes: 8 additions & 7 deletions pkg/config/storage.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,12 @@ package config

// Storage provides configs for various storage backends
type Storage struct {
Disk *DiskConfig
GCP *GCPConfig
Minio *MinioConfig
Mongo *MongoConfig
S3 *S3Config
AzureBlob *AzureBlobConfig
External *External
Disk *DiskConfig
GCP *GCPConfig
Minio *MinioConfig
Mongo *MongoConfig
S3 *S3Config
AzureBlob *AzureBlobConfig
External *External
Artifactory *ArtifactoryConfig
}
272 changes: 272 additions & 0 deletions pkg/storage/artifactory/client.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,272 @@
package artifactory

import (
"archive/zip"
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"net/http"
"net/url"
"path"
"strings"

"github.com/gomods/athens/pkg/config"
"github.com/gomods/athens/pkg/errors"
"github.com/gomods/athens/pkg/storage"

"golang.org/x/mod/module"
)

type service struct {
baseURL *url.URL
repo string
user string
pass string
c *http.Client
}

// New returns an artifactory storage client
func New(conf *config.ArtifactoryConfig, c *http.Client) (storage.Backend, error) {
const op errors.Op = "artifactory.New"
if c == nil {
c = &http.Client{}
}
u, err := url.Parse(conf.URL)
if err != nil {
return nil, errors.E(op, err)
}
var password string
switch {
case conf.Password != "":
password = conf.Password
case conf.APIKey != "":
password = conf.APIKey
case conf.AccessToken != "":
password = conf.AccessToken
}
return &service{
baseURL: u,
repo: conf.Repository,
user: conf.Username,
pass: password,
c: c,
}, nil
}

func (s *service) req(ctx context.Context, method string, u *url.URL, body io.Reader) (*http.Request, error) {
const op errors.Op = "artifactory.req"
req, err := http.NewRequest(method, s.baseURL.ResolveReference(u).String(), body)
if err != nil {
return nil, errors.E(op, err)
}
req.WithContext(ctx)
if s.pass != "" || s.user != "" {
req.SetBasicAuth(s.user, s.pass)
}
return req, nil
}

func (s *service) fileKey(mod, version, ext string) (string, error) {
const op errors.Op = "artifactory.fileKey"
var err error
mod, err = module.EscapePath(mod)
if err != nil {
return "", errors.E(op, err)
}
if version == "" {
return path.Join(s.repo, mod), nil
}
if ext == "" {
return path.Join(s.repo, mod, version), nil
}
return path.Join(s.repo, mod, version, ext), nil
}

func (s *service) getRequest(ctx context.Context, mod, version, ext string) (*http.Request, error) {
const op errors.Op = "artifactory.getRequest"
fileKey, err := s.fileKey(mod, version, ext)
if err != nil {
return nil, errors.E(op, err)
}
req, err := s.req(ctx, http.MethodGet, &url.URL{
Path: fileKey,
}, nil)
if err != nil {
return nil, errors.E(op, err)
}
return req, nil
}

func (s *service) do(req *http.Request) ([]byte, error) {
const op errors.Op = "artifactory.do"
resp, err := s.c.Do(req)
if err != nil {
return nil, errors.E(op, err)
}
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, errors.E(op, err)
}
if resp.StatusCode != 200 {
return nil, errors.E(op, fmt.Errorf("non 200 status code: %v - body: %s", resp.StatusCode, body), resp.StatusCode)
}
return body, nil
}

func (s *service) List(ctx context.Context, mod string) ([]string, error) {
const op errors.Op = "artifactory.List"
fileKey, err := s.fileKey(mod, "", "")
if err != nil {
return nil, errors.E(op, err)
}
req, err := s.req(ctx, http.MethodGet, &url.URL{
Path: path.Join("api", "storage", fileKey),
}, nil)
if err != nil {
return nil, errors.E(op, err)
}
req.Header.Add("Accept", "application/vnd.org.jfrog.artifactory.storage.FolderInfo+json")
resp, err := s.c.Do(req)
if err != nil {
return nil, errors.E(op, err)
}
defer resp.Body.Close()
if resp.StatusCode == 404 {
return nil, nil
} else if resp.StatusCode != 200 {
body, _ := ioutil.ReadAll(resp.Body)
return nil, errors.E(op, fmt.Errorf("non 200 status code: %v - body: %s", resp.StatusCode, body), resp.StatusCode)
}
var folderResp struct {
Children []struct {
URI string `json:"uri"`
Folder bool `json:"folder"`
} `json:"children"`
}
if err := json.NewDecoder(resp.Body).Decode(&folderResp); err != nil {
return nil, errors.E(op, err)
}
var versions []string
for _, child := range folderResp.Children {
if !child.Folder {
continue
}
version := strings.Trim(child.URI, "/")
versions = append(versions, version)
}
return versions, nil
}

func (s *service) Info(ctx context.Context, mod, ver string) ([]byte, error) {
const op errors.Op = "artifactory.Info"
req, err := s.getRequest(ctx, mod, ver, "mod.info")
if err != nil {
return nil, errors.E(op, err)
}
modFile, err := s.do(req)
if err != nil {
return nil, errors.E(op, err)
}
return modFile, nil
}

func (s *service) GoMod(ctx context.Context, mod, ver string) ([]byte, error) {
const op errors.Op = "artifactory.GoMod"
req, err := s.getRequest(ctx, mod, ver, "mod.mod")
if err != nil {
return nil, errors.E(op, err)
}
modFile, err := s.do(req)
if err != nil {
return nil, errors.E(op, err)
}
return modFile, nil
}

func (s *service) Zip(ctx context.Context, mod, ver string) (io.ReadCloser, error) {
const op errors.Op = "artifactory.Zip"
req, err := s.getRequest(ctx, mod, ver, "mod.zip")
if err != nil {
return nil, errors.E(op, err)
}
modFile, err := s.do(req)
if err != nil {
return nil, errors.E(op, err)
}
return ioutil.NopCloser(bytes.NewReader(modFile)), nil
}

func (s *service) Save(ctx context.Context, mod, ver string, modFile []byte, modZip io.Reader, info []byte) error {
const op errors.Op = "artifactory.Save"
var err error
fileKey, err := s.fileKey(mod, ver, ".zip")
if err != nil {
return errors.E(op, err)
}
pr, pw := io.Pipe()
zw := zip.NewWriter(pw)
go func() {
err := upload(zw, modFile, info, modZip)
pw.CloseWithError(err)
}()
req, err := s.req(ctx, http.MethodPut, &url.URL{
Path: fileKey,
}, pr)
if err != nil {
return errors.E(op, err)
}
req.Header.Add("Content-Type", "application/zip")
req.Header.Add("X-Explode-Archive", "true")
req.Header.Add("X-Explode-Archive-Atomic", "true")
if _, err := s.do(req); err != nil {
return errors.E(op, err)
}
return nil
}

func (s *service) Delete(ctx context.Context, mod, ver string) error {
const op errors.Op = "artifactory.Delete"
fileKey, err := s.fileKey(mod, ver, "")
if err != nil {
return errors.E(op, err)
}
req, err := s.req(ctx, http.MethodDelete, &url.URL{
Path: fileKey,
}, nil)
if _, err := s.do(req); err != nil {
return errors.E(op, err)
}
return nil
}

func upload(zw *zip.Writer, mod, info []byte, zip io.Reader) error {
defer zw.Close()
infoW, err := zw.Create("mod.info")
if err != nil {
return fmt.Errorf("error creating info file: %v", err)
}
_, err = infoW.Write(info)
if err != nil {
return fmt.Errorf("error writing info file: %v", err)
}
modW, err := zw.Create("mod.mod")
if err != nil {
return fmt.Errorf("error creating mod file: %v", err)
}
_, err = modW.Write(mod)
if err != nil {
return fmt.Errorf("error writing mod file: %v", err)
}
zipW, err := zw.Create("mod.zip")
if err != nil {
return fmt.Errorf("error creating zip file: %v", err)
}
_, err = io.Copy(zipW, zip)
if err != nil {
return fmt.Errorf("error writing zip file: %v", err)
}
return nil
}