client.go

// This file is part of MemoIO Console Server
// Copyright (c) 2021 MinIO, Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package restapi

import (
	"context"
	"fmt"
	"io"
	"net/url"
	"path"
	"strings"
	"time"

	"github.com/minio/minio-go/v7/pkg/replication"
	"github.com/minio/minio-go/v7/pkg/sse"

	"errors"

	"github.com/memoio/console/models"
	"github.com/memoio/console/pkg"
	"github.com/memoio/console/pkg/auth"
	"github.com/memoio/console/pkg/auth/ldap"
	xjwt "github.com/memoio/console/pkg/auth/token"
	mc "github.com/minio/mc/cmd"
	"github.com/minio/mc/pkg/probe"
	"github.com/minio/minio-go/v7"
	"github.com/minio/minio-go/v7/pkg/credentials"
	"github.com/minio/minio-go/v7/pkg/lifecycle"
	"github.com/minio/minio-go/v7/pkg/notification"
	"github.com/minio/minio-go/v7/pkg/tags"
)

func init() {
	// All minio-go API operations shall be performed only once,
	// another way to look at this is we are turning off retries.
	minio.MaxRetry = 1
}

// MinioClient interface with all functions to be implemented
// by mock when testing, it should include all MinioClient respective api calls
// that are used within this project.
type MinioClient interface {
	listBucketsWithContext(ctx context.Context) ([]minio.BucketInfo, error)
	makeBucketWithContext(ctx context.Context, bucketName, location string, objectLocking bool) error
	setBucketPolicyWithContext(ctx context.Context, bucketName, policy string) error
	removeBucket(ctx context.Context, bucketName string) error
	getBucketNotification(ctx context.Context, bucketName string) (config notification.Configuration, err error)
	getBucketPolicy(ctx context.Context, bucketName string) (string, error)
	listObjects(ctx context.Context, bucket string, opts minio.ListObjectsOptions) <-chan minio.ObjectInfo
	getObjectRetention(ctx context.Context, bucketName, objectName, versionID string) (mode *minio.RetentionMode, retainUntilDate *time.Time, err error)
	getObjectLegalHold(ctx context.Context, bucketName, objectName string, opts minio.GetObjectLegalHoldOptions) (status *minio.LegalHoldStatus, err error)
	putObject(ctx context.Context, bucketName, objectName string, reader io.Reader, objectSize int64, opts minio.PutObjectOptions) (info minio.UploadInfo, err error)
	putObjectLegalHold(ctx context.Context, bucketName, objectName string, opts minio.PutObjectLegalHoldOptions) error
	putObjectRetention(ctx context.Context, bucketName, objectName string, opts minio.PutObjectRetentionOptions) error
	statObject(ctx context.Context, bucketName, prefix string, opts minio.GetObjectOptions) (objectInfo minio.ObjectInfo, err error)
	setBucketEncryption(ctx context.Context, bucketName string, config *sse.Configuration) error
	removeBucketEncryption(ctx context.Context, bucketName string) error
	getBucketEncryption(ctx context.Context, bucketName string) (*sse.Configuration, error)
	putObjectTagging(ctx context.Context, bucketName, objectName string, otags *tags.Tags, opts minio.PutObjectTaggingOptions) error
	getObjectTagging(ctx context.Context, bucketName, objectName string, opts minio.GetObjectTaggingOptions) (*tags.Tags, error)
	setObjectLockConfig(ctx context.Context, bucketName string, mode *minio.RetentionMode, validity *uint, unit *minio.ValidityUnit) error
	getBucketObjectLockConfig(ctx context.Context, bucketName string) (mode *minio.RetentionMode, validity *uint, unit *minio.ValidityUnit, err error)
	getObjectLockConfig(ctx context.Context, bucketName string) (lock string, mode *minio.RetentionMode, validity *uint, unit *minio.ValidityUnit, err error)
	getLifecycleRules(ctx context.Context, bucketName string) (lifecycle *lifecycle.Configuration, err error)
	setBucketLifecycle(ctx context.Context, bucketName string, config *lifecycle.Configuration) error
	copyObject(ctx context.Context, dst minio.CopyDestOptions, src minio.CopySrcOptions) (minio.UploadInfo, error)
	GetBucketTagging(ctx context.Context, bucketName string) (*tags.Tags, error)
	SetBucketTagging(ctx context.Context, bucketName string, tags *tags.Tags) error
	RemoveBucketTagging(ctx context.Context, bucketName string) error
}

// Interface implementation
//
// Define the structure of a minIO Client and define the functions that are actually used
// from minIO api.
type minioClient struct {
	client *minio.Client
}

func (c minioClient) GetBucketTagging(ctx context.Context, bucketName string) (*tags.Tags, error) {
	return c.client.GetBucketTagging(ctx, bucketName)
}

func (c minioClient) SetBucketTagging(ctx context.Context, bucketName string, tags *tags.Tags) error {
	return c.client.SetBucketTagging(ctx, bucketName, tags)
}

func (c minioClient) RemoveBucketTagging(ctx context.Context, bucketName string) error {
	return c.client.RemoveBucketTagging(ctx, bucketName)
}

// implements minio.ListBuckets(ctx)
func (c minioClient) listBucketsWithContext(ctx context.Context) ([]minio.BucketInfo, error) {
	return c.client.ListBuckets(ctx)
}

// implements minio.MakeBucketWithContext(ctx, bucketName, location, objectLocking)
func (c minioClient) makeBucketWithContext(ctx context.Context, bucketName, location string, objectLocking bool) error {
	return c.client.MakeBucket(ctx, bucketName, minio.MakeBucketOptions{
		Region:        location,
		ObjectLocking: objectLocking,
	})
}

// implements minio.SetBucketPolicyWithContext(ctx, bucketName, policy)
func (c minioClient) setBucketPolicyWithContext(ctx context.Context, bucketName, policy string) error {
	return c.client.SetBucketPolicy(ctx, bucketName, policy)
}

// implements minio.RemoveBucket(bucketName)
func (c minioClient) removeBucket(ctx context.Context, bucketName string) error {
	return c.client.RemoveBucket(ctx, bucketName)
}

// implements minio.GetBucketNotification(bucketName)
func (c minioClient) getBucketNotification(ctx context.Context, bucketName string) (config notification.Configuration, err error) {
	return c.client.GetBucketNotification(ctx, bucketName)
}

// implements minio.GetBucketPolicy(bucketName)
func (c minioClient) getBucketPolicy(ctx context.Context, bucketName string) (string, error) {
	return c.client.GetBucketPolicy(ctx, bucketName)
}

// implements minio.getBucketVersioning(ctx, bucketName)
func (c minioClient) getBucketVersioning(ctx context.Context, bucketName string) (minio.BucketVersioningConfiguration, error) {
	return c.client.GetBucketVersioning(ctx, bucketName)
}

// implements minio.getBucketVersioning(ctx, bucketName)
func (c minioClient) getBucketReplication(ctx context.Context, bucketName string) (replication.Config, error) {
	return c.client.GetBucketReplication(ctx, bucketName)
}

// implements minio.listObjects(ctx)
func (c minioClient) listObjects(ctx context.Context, bucket string, opts minio.ListObjectsOptions) <-chan minio.ObjectInfo {
	return c.client.ListObjects(ctx, bucket, opts)
}

func (c minioClient) getObjectRetention(ctx context.Context, bucketName, objectName, versionID string) (mode *minio.RetentionMode, retainUntilDate *time.Time, err error) {
	return c.client.GetObjectRetention(ctx, bucketName, objectName, versionID)
}

func (c minioClient) getObjectLegalHold(ctx context.Context, bucketName, objectName string, opts minio.GetObjectLegalHoldOptions) (status *minio.LegalHoldStatus, err error) {
	return c.client.GetObjectLegalHold(ctx, bucketName, objectName, opts)
}

func (c minioClient) putObject(ctx context.Context, bucketName, objectName string, reader io.Reader, objectSize int64, opts minio.PutObjectOptions) (info minio.UploadInfo, err error) {
	return c.client.PutObject(ctx, bucketName, objectName, reader, objectSize, opts)
}

func (c minioClient) putObjectLegalHold(ctx context.Context, bucketName, objectName string, opts minio.PutObjectLegalHoldOptions) error {
	return c.client.PutObjectLegalHold(ctx, bucketName, objectName, opts)
}

func (c minioClient) putObjectRetention(ctx context.Context, bucketName, objectName string, opts minio.PutObjectRetentionOptions) error {
	return c.client.PutObjectRetention(ctx, bucketName, objectName, opts)
}

func (c minioClient) statObject(ctx context.Context, bucketName, prefix string, opts minio.GetObjectOptions) (objectInfo minio.ObjectInfo, err error) {
	return c.client.StatObject(ctx, bucketName, prefix, opts)
}

// implements minio.SetBucketEncryption(ctx, bucketName, config)
func (c minioClient) setBucketEncryption(ctx context.Context, bucketName string, config *sse.Configuration) error {
	return c.client.SetBucketEncryption(ctx, bucketName, config)
}

// implements minio.RemoveBucketEncryption(ctx, bucketName)
func (c minioClient) removeBucketEncryption(ctx context.Context, bucketName string) error {
	return c.client.RemoveBucketEncryption(ctx, bucketName)
}

// implements minio.GetBucketEncryption(ctx, bucketName, config)
func (c minioClient) getBucketEncryption(ctx context.Context, bucketName string) (*sse.Configuration, error) {
	return c.client.GetBucketEncryption(ctx, bucketName)
}

func (c minioClient) putObjectTagging(ctx context.Context, bucketName, objectName string, otags *tags.Tags, opts minio.PutObjectTaggingOptions) error {
	return c.client.PutObjectTagging(ctx, bucketName, objectName, otags, opts)
}

func (c minioClient) getObjectTagging(ctx context.Context, bucketName, objectName string, opts minio.GetObjectTaggingOptions) (*tags.Tags, error) {
	return c.client.GetObjectTagging(ctx, bucketName, objectName, opts)
}

func (c minioClient) setObjectLockConfig(ctx context.Context, bucketName string, mode *minio.RetentionMode, validity *uint, unit *minio.ValidityUnit) error {
	return c.client.SetObjectLockConfig(ctx, bucketName, mode, validity, unit)
}

func (c minioClient) getBucketObjectLockConfig(ctx context.Context, bucketName string) (mode *minio.RetentionMode, validity *uint, unit *minio.ValidityUnit, err error) {
	return c.client.GetBucketObjectLockConfig(ctx, bucketName)
}

func (c minioClient) getObjectLockConfig(ctx context.Context, bucketName string) (lock string, mode *minio.RetentionMode, validity *uint, unit *minio.ValidityUnit, err error) {
	return c.client.GetObjectLockConfig(ctx, bucketName)
}

func (c minioClient) getLifecycleRules(ctx context.Context, bucketName string) (lifecycle *lifecycle.Configuration, err error) {
	return c.client.GetBucketLifecycle(ctx, bucketName)
}

func (c minioClient) setBucketLifecycle(ctx context.Context, bucketName string, config *lifecycle.Configuration) error {
	return c.client.SetBucketLifecycle(ctx, bucketName, config)
}

func (c minioClient) copyObject(ctx context.Context, dst minio.CopyDestOptions, src minio.CopySrcOptions) (minio.UploadInfo, error) {
	return c.client.CopyObject(ctx, dst, src)
}

// MCClient interface with all functions to be implemented
// by mock when testing, it should include all mc/S3Client respective api calls
// that are used within this project.
type MCClient interface {
	addNotificationConfig(ctx context.Context, arn string, events []string, prefix, suffix string, ignoreExisting bool) *probe.Error
	removeNotificationConfig(ctx context.Context, arn string, event string, prefix string, suffix string) *probe.Error
	watch(ctx context.Context, options mc.WatchOptions) (*mc.WatchObject, *probe.Error)
	remove(ctx context.Context, isIncomplete, isRemoveBucket, isBypass bool, contentCh <-chan *mc.ClientContent) <-chan mc.RemoveResult
	list(ctx context.Context, opts mc.ListOptions) <-chan *mc.ClientContent
	get(ctx context.Context, opts mc.GetOptions) (io.ReadCloser, *probe.Error)
	shareDownload(ctx context.Context, versionID string, expires time.Duration) (string, *probe.Error)
	setVersioning(ctx context.Context, status string) *probe.Error
}

// Interface implementation
//
// Define the structure of a mc S3Client and define the functions that are actually used
// from mcS3client api.
type mcClient struct {
	client *mc.S3Client
}

// implements S3Client.AddNotificationConfig()
func (c mcClient) addNotificationConfig(ctx context.Context, arn string, events []string, prefix, suffix string, ignoreExisting bool) *probe.Error {
	return c.client.AddNotificationConfig(ctx, arn, events, prefix, suffix, ignoreExisting)
}

// implements S3Client.RemoveNotificationConfig()
func (c mcClient) removeNotificationConfig(ctx context.Context, arn string, event string, prefix string, suffix string) *probe.Error {
	return c.client.RemoveNotificationConfig(ctx, arn, event, prefix, suffix)
}

func (c mcClient) watch(ctx context.Context, options mc.WatchOptions) (*mc.WatchObject, *probe.Error) {
	return c.client.Watch(ctx, options)
}

func (c mcClient) setReplication(ctx context.Context, cfg *replication.Config, opts replication.Options) *probe.Error {
	return c.client.SetReplication(ctx, cfg, opts)
}

func (c mcClient) deleteAllReplicationRules(ctx context.Context) *probe.Error {
	return c.client.RemoveReplication(ctx)
}

func (c mcClient) setVersioning(ctx context.Context, status string) *probe.Error {
	return c.client.SetVersion(ctx, status)
}

func (c mcClient) remove(ctx context.Context, isIncomplete, isRemoveBucket, isBypass bool, contentCh <-chan *mc.ClientContent) <-chan mc.RemoveResult {
	return c.client.Remove(ctx, isIncomplete, isRemoveBucket, isBypass, contentCh)
}

func (c mcClient) list(ctx context.Context, opts mc.ListOptions) <-chan *mc.ClientContent {
	return c.client.List(ctx, opts)
}

func (c mcClient) get(ctx context.Context, opts mc.GetOptions) (io.ReadCloser, *probe.Error) {
	return c.client.Get(ctx, opts)
}

func (c mcClient) shareDownload(ctx context.Context, versionID string, expires time.Duration) (string, *probe.Error) {
	return c.client.ShareDownload(ctx, versionID, expires)
}

// ConsoleCredentialsI interface with all functions to be implemented
// by mock when testing, it should include all needed consoleCredentials.Login api calls
// that are used within this project.
type ConsoleCredentialsI interface {
	Get() (credentials.Value, error)
	Expire()
	GetAccountAccessKey() string
}

// Interface implementation
type ConsoleCredentials struct {
	ConsoleCredentials *credentials.Credentials
	AccountAccessKey   string
}

func (c ConsoleCredentials) GetAccountAccessKey() string {
	return c.AccountAccessKey
}

// Get implements *Login.Get()
func (c ConsoleCredentials) Get() (credentials.Value, error) {
	return c.ConsoleCredentials.Get()
}

// Expire implements *Login.Expire()
func (c ConsoleCredentials) Expire() {
	c.ConsoleCredentials.Expire()
}

// consoleSTSAssumeRole it's a STSAssumeRole wrapper, in general
// there's no need to use this struct anywhere else in the project, it's only required
// for passing a custom *http.Client to *credentials.STSAssumeRole
type consoleSTSAssumeRole struct {
	stsAssumeRole *credentials.STSAssumeRole
}

func (s consoleSTSAssumeRole) Retrieve() (credentials.Value, error) {
	return s.stsAssumeRole.Retrieve()
}

func (s consoleSTSAssumeRole) IsExpired() bool {
	return s.stsAssumeRole.IsExpired()
}

func NewConsoleCredentials(accessKey, secretKey, location string) (*credentials.Credentials, error) {
	// Future authentication methods can be added under this switch statement
	switch {
	// LDAP authentication for Console
	case ldap.GetLDAPEnabled():
		{
			creds, err := auth.GetCredentialsFromLDAP(GetConsoleHTTPClient(), getMinIOServer(), accessKey, secretKey)
			if err != nil {
				return nil, err
			}
			return creds, nil
		}
	// default authentication for Console is via STS (Security Token Service) against MinIO
	default:
		{
			if accessKey == "" || secretKey == "" {
				return nil, errors.New("credentials endpoint, access and secret key are mandatory for AssumeRoleSTS")
			}
			opts := credentials.STSAssumeRoleOptions{
				AccessKey:       accessKey,
				SecretKey:       secretKey,
				Location:        location,
				DurationSeconds: int(xjwt.GetConsoleSTSDuration().Seconds()),
			}
			stsAssumeRole := &credentials.STSAssumeRole{
				Client:      GetConsoleHTTPClient(),
				STSEndpoint: getMinIOServer(),
				Options:     opts,
			}
			consoleSTSWrapper := consoleSTSAssumeRole{stsAssumeRole: stsAssumeRole}
			return credentials.New(consoleSTSWrapper), nil
		}
	}
}

// getConsoleCredentialsFromSession returns the *consoleCredentials.Login associated to the
// provided session token, this is useful for running the Expire() or IsExpired() operations
func getConsoleCredentialsFromSession(claims *models.Principal) *credentials.Credentials {
	return credentials.NewStaticV4(claims.STSAccessKeyID, claims.STSSecretAccessKey, claims.STSSessionToken)
}

// newMinioClient creates a new MinIO client based on the ConsoleCredentials extracted
// from the provided session token
func newMinioClient(claims *models.Principal) (*minio.Client, error) {
	creds := getConsoleCredentialsFromSession(claims)
	minioClient, err := minio.New(getMinIOEndpoint(), &minio.Options{
		Creds:     creds,
		Secure:    getMinIOEndpointIsSecure(),
		Transport: GetConsoleHTTPClient().Transport,
	})
	if err != nil {
		return nil, err
	}
	return minioClient, nil
}

// computeObjectURLWithoutEncode returns a MinIO url containing the object filename without encoding
func computeObjectURLWithoutEncode(bucketName, prefix string) (string, error) {
	endpoint := getMinIOServer()
	u, err := url.Parse(endpoint)
	if err != nil {
		return "", fmt.Errorf("the provided endpoint is invalid")
	}
	objectURL := fmt.Sprintf("%s:%s", u.Hostname(), u.Port())
	if strings.TrimSpace(bucketName) != "" {
		objectURL = path.Join(objectURL, bucketName)
	}
	if strings.TrimSpace(prefix) != "" {
		objectURL = pathJoinFinalSlash(objectURL, prefix)
	}

	objectURL = fmt.Sprintf("%s://%s", u.Scheme, objectURL)
	return objectURL, nil
}

// newS3BucketClient creates a new mc S3Client to talk to the server based on a bucket
func newS3BucketClient(claims *models.Principal, bucketName string, prefix string) (*mc.S3Client, error) {
	if claims == nil {
		return nil, fmt.Errorf("the provided credentials are invalid")
	}
	// It's very important to avoid encoding the prefix since the minio client will encode the path itself
	objectURL, err := computeObjectURLWithoutEncode(bucketName, prefix)
	if err != nil {
		return nil, fmt.Errorf("the provided endpoint is invalid")
	}
	s3Config := newS3Config(objectURL, claims.STSAccessKeyID, claims.STSSecretAccessKey, claims.STSSessionToken, false)
	client, pErr := mc.S3New(s3Config)
	if pErr != nil {
		return nil, pErr.Cause
	}
	s3Client, ok := client.(*mc.S3Client)
	if !ok {
		return nil, fmt.Errorf("the provided url doesn't point to a S3 server")
	}
	return s3Client, nil
}

// pathJoinFinalSlash - like path.Join() but retains trailing slashSeparator of the last element
func pathJoinFinalSlash(elem ...string) string {
	if len(elem) > 0 {
		if strings.HasSuffix(elem[len(elem)-1], SlashSeparator) {
			return path.Join(elem...) + SlashSeparator
		}
	}
	return path.Join(elem...)
}

// newS3Config simply creates a new Config struct using the passed
// parameters.
func newS3Config(endpoint, accessKey, secretKey, sessionToken string, insecure bool) *mc.Config {
	// We have a valid alias and hostConfig. We populate the/
	// consoleCredentials from the match found in the config file.
	s3Config := new(mc.Config)

	s3Config.AppName = globalAppName
	s3Config.AppVersion = pkg.Version
	s3Config.Debug = false
	s3Config.Insecure = insecure

	s3Config.HostURL = endpoint
	s3Config.AccessKey = accessKey
	s3Config.SecretKey = secretKey
	s3Config.SessionToken = sessionToken
	s3Config.Signature = "S3v4"
	s3Config.Transport = prepareSTSClientTransport(insecure)

	return s3Config
}