object.go

/******************************************************************************
*
*  Copyright 2018 Stefan Majewsky <majewsky@gmx.net>
*
*  Licensed under the Apache License, Version 2.0 (the "License");
*  you may not use this file except in compliance with the License.
*  You may obtain a copy of the License at
*
*      http://www.apache.org/licenses/LICENSE-2.0
*
*  Unless required by applicable law or agreed to in writing, software
*  distributed under the License is distributed on an "AS IS" BASIS,
*  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*  See the License for the specific language governing permissions and
*  limitations under the License.
*
******************************************************************************/

package schwift

import (
	"bytes"
	"crypto/hmac"
	"crypto/md5"
	"crypto/sha1"
	"encoding/hex"
	"fmt"
	"hash"
	"io"
	"net/http"
	"net/url"
	"strings"
	"time"
)

//Object represents a Swift object. Instances are usually obtained by
//traversing downwards from a container with Container.Object() or
//Container.Objects().
type Object struct {
	c    *Container
	name string
	//cache
	headers        *ObjectHeaders //from HEAD/GET without ?symlink=get
	symlinkHeaders *ObjectHeaders //from HEAD/GET with ?symlink=get
}

//IsEqualTo returns true if both Object instances refer to the same object.
func (o *Object) IsEqualTo(other *Object) bool {
	return other.name == o.name && other.c.IsEqualTo(o.c)
}

//Object returns a handle to the object with the given name within this
//container. This function does not issue any HTTP requests, and therefore cannot
//ensure that the object exists. Use the Exists() function to check for the
//object's existence.
func (c *Container) Object(name string) *Object {
	return &Object{c: c, name: name}
}

//Container returns a handle to the container this object is stored in.
func (o *Object) Container() *Container {
	return o.c
}

//Name returns the object name. This does not parse the name in any way; if you
//want only the basename portion of the object name, use package path from the
//standard library in conjunction with this function. For example:
//
//	obj := account.Container("docs").Object("2018-02-10/invoice.pdf")
//	obj.Name()            //returns "2018-02-10/invoice.pdf"
//	path.Base(obj.Name()) //returns            "invoice.pdf"
func (o *Object) Name() string {
	return o.name
}

//FullName returns the container name and object name joined together with a
//slash. This identifier is used by Swift in several places (large object
//manifests, symlink targets, etc.) to refer to an object within an account.
//For example:
//
//	obj := account.Container("docs").Object("2018-02-10/invoice.pdf")
//	obj.Name()     //returns      "2018-02-10/invoice.pdf"
//	obj.FullName() //returns "docs/2018-02-10/invoice.pdf"
func (o *Object) FullName() string {
	return o.c.name + "/" + o.name
}

//Exists checks if this object exists, potentially by issuing a HEAD request
//if no Headers() have been cached yet.
func (o *Object) Exists() (bool, error) {
	_, err := o.Headers()
	if Is(err, http.StatusNotFound) {
		return false, nil
	} else if err != nil {
		return false, err
	}
	return true, nil
}

//Headers returns the ObjectHeaders for this object. If the ObjectHeaders
//has not been cached yet, a HEAD request is issued on the object.
//
//For symlinks, this operation returns the metadata for the target object. Use
//Object.SymlinkHeaders() to obtain the metadata for the symlink instead.
//
//This operation fails with http.StatusNotFound if the object does not exist.
//
//WARNING: This method is not thread-safe. Calling it concurrently on the same
//object results in undefined behavior.
func (o *Object) Headers() (ObjectHeaders, error) {
	if o.headers != nil {
		return *o.headers, nil
	}

	hdr, err := o.fetchHeaders(nil)
	if err != nil {
		return ObjectHeaders{}, err
	}
	o.headers = hdr
	return *hdr, nil
}

func (o *Object) fetchHeaders(opts *RequestOptions) (*ObjectHeaders, error) {
	resp, err := Request{
		Method:        "HEAD",
		ContainerName: o.c.name,
		ObjectName:    o.name,
		Options:       opts,
		//since Openstack LOVES to be inconsistent with everything (incl. itself),
		//this returns 200 instead of 204
		ExpectStatusCodes: []int{200},
		DrainResponseBody: true,
	}.Do(o.c.a.backend)
	if err != nil {
		return nil, err
	}

	headers := ObjectHeaders{headersFromHTTP(resp.Header)}
	return &headers, headers.Validate()
}

//Update updates the object's headers using a POST request. To add URL
//parameters, pass a non-nil *RequestOptions.
//
//This operation fails with http.StatusNotFound if the object does not exist.
//
//A successful POST request implies Invalidate() since it may change metadata.
func (o *Object) Update(headers ObjectHeaders, opts *RequestOptions) error {
	_, err := Request{
		Method:            "POST",
		ContainerName:     o.c.name,
		ObjectName:        o.name,
		Options:           cloneRequestOptions(opts, headers.Headers),
		ExpectStatusCodes: []int{202},
	}.Do(o.c.a.backend)
	if err == nil {
		o.Invalidate()
	}
	return err
}

//UploadOptions invokes advanced behavior in the Object.Upload() method.
type UploadOptions struct {
	//When overwriting a large object, delete its segments. This will cause
	//Upload() to call into BulkDelete(), so a BulkError may be returned.
	DeleteSegments bool
}

//Upload creates the object using a PUT request.
//
//If you do not have an io.Reader, but you have a []byte or string instance
//containing the object, wrap it in a *bytes.Reader instance like so:
//
//	var buffer []byte
//	o.Upload(bytes.NewReader(buffer), opts)
//
//	//or...
//	var buffer string
//	o.Upload(bytes.NewReader([]byte(buffer)), opts)
//
//If you have neither an io.Reader nor a []byte or string, but you have a
//function that generates the object's content into an io.Writer, use
//UploadFromWriter instead.
//
//If the object is very large and you want to upload it in segments, use
//LargeObject.Append() instead. See documentation on type LargeObject for
//details.
//
//If content is a *bytes.Reader or a *bytes.Buffer instance, the Content-Length
//and Etag request headers will be computed automatically. Otherwise, it is
//highly recommended that the caller set these headers (if possible) to allow
//the server to check the integrity of the uploaded file.
//
//If Etag and/or Content-Length is supplied and the content does not match
//these parameters, http.StatusUnprocessableEntity is returned. If Etag is not
//supplied and cannot be computed in advance, Upload() will compute the Etag as
//data is read from the io.Reader, and compare the result to the Etag returned
//by Swift, returning ErrChecksumMismatch in case of mismatch. The object will
//have been uploaded at that point, so you will usually want to Delete() it.
//
//This function can be used regardless of whether the object exists or not.
//
//A successful PUT request implies Invalidate() since it may change metadata.
func (o *Object) Upload(content io.Reader, opts *UploadOptions, ropts *RequestOptions) error {
	if opts == nil {
		opts = &UploadOptions{}
	}

	ropts = cloneRequestOptions(ropts, nil)
	hdr := ObjectHeaders{ropts.Headers}

	if !hdr.SizeBytes().Exists() {
		value := tryComputeContentLength(content)
		if value != nil {
			hdr.SizeBytes().Set(*value)
		}
	}

	//do not attempt to add the Etag header when we're writing a large object
	//manifest; the header refers to the content, but we would be computing the
	//manifest's hash instead
	isManifestUpload := ropts.Values.Get("multipart-manifest") == "put" || hdr.IsDynamicLargeObject()

	var hasher hash.Hash
	if !isManifestUpload {
		tryComputeEtag(content, hdr)

		//could not compute Etag in advance -> need to check on the fly
		if !hdr.Etag().Exists() {
			hasher = md5.New()
			if content != nil {
				content = io.TeeReader(content, hasher)
			}
		}
	}

	var lo *LargeObject
	if opts.DeleteSegments {
		//enumerate segments in large object before overwriting it, but only delete
		//the segments after successfully uploading the new object to decrease the
		//chance of an inconsistent state following an upload error
		var err error
		lo, err = o.AsLargeObject()
		switch err {
		case nil:
			//okay, delete segments at the end
		case ErrNotLarge:
			//okay, do not try to delete segments
			lo = nil
		default:
			//unexpected error
			return err
		}
	}

	resp, err := Request{
		Method:            "PUT",
		ContainerName:     o.c.name,
		ObjectName:        o.name,
		Options:           ropts,
		Body:              content,
		ExpectStatusCodes: []int{201},
		DrainResponseBody: true,
	}.Do(o.c.a.backend)
	if err != nil {
		return err
	}
	o.Invalidate()

	if hasher != nil {
		expectedEtag := hex.EncodeToString(hasher.Sum(nil))
		if expectedEtag != resp.Header.Get("Etag") {
			return ErrChecksumMismatch
		}
	}

	if opts.DeleteSegments && lo != nil {
		_, _, err := lo.object.c.a.BulkDelete(lo.SegmentObjects(), nil, nil)
		if err != nil {
			return err
		}
	}

	return nil
}

type readerWithLen interface {
	//Returns the number of bytes in the unread portion of the buffer.
	//Implemented by bytes.Reader, bytes.Buffer and strings.Reader.
	Len() int
}

func tryComputeContentLength(content io.Reader) *uint64 {
	if content == nil {
		val := uint64(0)
		return &val
	} else if r, ok := content.(readerWithLen); ok {
		val := uint64(r.Len())
		return &val
	}
	return nil
}

func tryComputeEtag(content io.Reader, headers ObjectHeaders) {
	h := headers.Etag()
	if h.Exists() {
		return
	}
	switch r := content.(type) {
	case nil:
		sum := md5.Sum(nil)
		h.Set(hex.EncodeToString(sum[:]))
	case *bytes.Buffer:
		//bytes.Buffer has a method that returns the unread portion of the buffer,
		//so this one is easy
		sum := md5.Sum(r.Bytes())
		h.Set(hex.EncodeToString(sum[:]))
	case io.ReadSeeker:
		//bytes.Reader does not have such a method, but it is an io.Seeker, so we
		//can read the entire thing and then seek back to where we started
		hash := md5.New()
		n, _ := io.Copy(hash, r)
		r.Seek(-n, io.SeekCurrent)
		h.Set(hex.EncodeToString(hash.Sum(nil)))
	}
}

//UploadFromWriter is a variant of Upload that can be used when the object's
//content is generated by some function or package that takes an io.Writer
//instead of supplying an io.Reader. For example:
//
//	func greeting(target io.Writer, name string) error {
//	    _, err := fmt.Fprintf(target, "Hello %s!\n", name)
//	    return err
//	}
//
//	obj := container.Object("greeting-for-susan-and-jeffrey")
//	err := obj.UploadFromWriter(nil, func(w io.Writer) error {
//	    err := greeting(w, "Susan")
//	    if err == nil {
//	        err = greeting(w, "Jeffrey")
//	    }
//	    return err
//	})
//
//If you do not need an io.Writer, always use Upload instead.
func (o *Object) UploadFromWriter(opts *UploadOptions, ropts *RequestOptions, callback func(io.Writer) error) error {
	reader, writer := io.Pipe()
	errChan := make(chan error)
	go func() {
		err := o.Upload(reader, opts, ropts)
		reader.CloseWithError(err) //stop the writer if it is still writing
		errChan <- err
	}()
	writer.CloseWithError(callback(writer)) //stop the reader if it is still reading
	return <-errChan
}

//DeleteOptions invokes advanced behavior in the Object.Delete() method.
type DeleteOptions struct {
	//When deleting a large object, also delete its segments. This will cause
	//Delete() to call into BulkDelete(), so a BulkError may be returned.
	DeleteSegments bool
}

//Delete deletes the object using a DELETE request. To add URL parameters,
//pass a non-nil *RequestOptions.
//
//This operation fails with http.StatusNotFound if the object does not exist.
//
//A successful DELETE request implies Invalidate().
func (o *Object) Delete(opts *DeleteOptions, ropts *RequestOptions) error {
	if opts == nil {
		opts = &DeleteOptions{}
	}
	if opts.DeleteSegments {
		exists, err := o.Exists()
		if err != nil {
			return err
		}
		if exists {
			lo, err := o.AsLargeObject()
			switch err {
			case nil:
				//is large object - delete segments and the object itself in one step
				_, _, err := o.c.a.BulkDelete(append(lo.SegmentObjects(), o), nil, nil)
				o.Invalidate()
				return err
			case ErrNotLarge:
				//not a large object - use regular DELETE request
			default:
				//unexpected error
				return err
			}
		}
	}

	_, err := Request{
		Method:            "DELETE",
		ContainerName:     o.c.name,
		ObjectName:        o.name,
		Options:           ropts,
		ExpectStatusCodes: []int{204},
	}.Do(o.c.a.backend)
	if err == nil {
		o.Invalidate()
	}
	return err
}

//Invalidate clears the internal cache of this Object instance. The next call
//to Headers() on this instance will issue a HEAD request on the object.
//
//WARNING: This method is not thread-safe. Calling it concurrently on the same
//object results in undefined behavior.
func (o *Object) Invalidate() {
	o.headers = nil
	o.symlinkHeaders = nil
}

//Download retrieves the object's contents using a GET request. This returns a
//helper object which allows you to select whether you want an io.ReadCloser
//for reading the object contents progressively, or whether you want the object
//contents collected into a byte slice or string.
//
//	reader, err := object.Download(nil).AsReadCloser()
//
//	buf, err := object.Download(nil).AsByteSlice()
//
//	str, err := object.Download(nil).AsString()
//
//See documentation on type DownloadedObject for details.
//
//WARNING: This method is not thread-safe. Calling it concurrently on the same
//object results in undefined behavior.
func (o *Object) Download(opts *RequestOptions) DownloadedObject {
	resp, err := Request{
		Method:            "GET",
		ContainerName:     o.c.name,
		ObjectName:        o.name,
		Options:           opts,
		ExpectStatusCodes: []int{200},
	}.Do(o.c.a.backend)
	var body io.ReadCloser
	if err == nil {
		newHeaders := ObjectHeaders{headersFromHTTP(resp.Header)}
		err = newHeaders.Validate()
		if err == nil {
			if opts != nil && opts.Values != nil && opts.Values.Get("symlink") == "get" {
				o.symlinkHeaders = &newHeaders
			} else {
				o.headers = &newHeaders
			}
		}
		body = resp.Body
	}
	return DownloadedObject{body, err}
}

//CopyOptions invokes advanced behavior in the Object.Copy() method.
type CopyOptions struct {
	//Copy only the object's content, not its metadata. New metadata can always
	//be supplied in the RequestOptions argument of Object.CopyTo().
	FreshMetadata bool
	//When the source is a symlink, copy the symlink instead of the target object.
	ShallowCopySymlinks bool
}

//CopyTo copies the object on the server side using a COPY request.
//
//A successful COPY implies target.Invalidate() since it may change the
//target's metadata.
func (o *Object) CopyTo(target *Object, opts *CopyOptions, ropts *RequestOptions) error {
	ropts = cloneRequestOptions(ropts, nil)
	ropts.Headers.Set("Destination", target.FullName())
	if o.c.a.name != target.c.a.name {
		ropts.Headers.Set("Destination-Account", target.c.a.name)
	}
	if opts != nil {
		if opts.FreshMetadata {
			ropts.Headers.Set("X-Fresh-Metadata", "true")
		}
		if opts.ShallowCopySymlinks {
			ropts.Values.Set("symlink", "get")
		}
	}

	_, err := Request{
		Method:            "COPY",
		ContainerName:     o.c.name,
		ObjectName:        o.name,
		Options:           ropts,
		ExpectStatusCodes: []int{201},
		DrainResponseBody: true,
	}.Do(o.c.a.backend)
	if err == nil {
		target.Invalidate()
	}
	return err
}

//SymlinkOptions invokes advanced behavior in the Object.SymlinkTo() method.
type SymlinkOptions struct {
	//When overwriting a large object, delete its segments. This will cause
	//SymlinkTo() to call into BulkDelete(), so a BulkError may be returned.
	DeleteSegments bool
}

//SymlinkTo creates the object as a symbolic link to another object using a PUT
//request. Like Object.Upload(), this method works regardless of whether the
//object already exists or not. Existing object contents will be overwritten by
//this operation.
//
//A successful PUT request implies Invalidate() since it may change metadata.
func (o *Object) SymlinkTo(target *Object, opts *SymlinkOptions, ropts *RequestOptions) error {
	ropts = cloneRequestOptions(ropts, nil)
	ropts.Headers.Set("X-Symlink-Target", target.FullName())
	if !target.c.a.IsEqualTo(o.c.a) {
		ropts.Headers.Set("X-Symlink-Target-Account", target.c.a.Name())
	}
	if ropts.Headers.Get("Content-Type") == "" {
		//recommended Content-Type for symlinks as per
		//<https://docs.openstack.org/swift/latest/middleware.html#symlink>
		ropts.Headers.Set("Content-Type", "application/symlink")
	}

	var uopts *UploadOptions
	if opts != nil {
		uopts = &UploadOptions{
			DeleteSegments: opts.DeleteSegments,
		}
	}

	return o.Upload(nil, uopts, ropts)
}

//SymlinkHeaders is similar to Headers, but if the object is a symlink, it
//returns the metadata of the symlink rather than the metadata of the target.
//It also returns a reference to the target object.
//
//If this object is not a symlink, Object.SymlinkHeaders() returns the same
//ObjectHeaders as Object.Headers(), and a nil target object.
//
//In a nutshell, if Object.Headers() is like os.Stat(), then
//Object.SymlinkHeaders() is like os.Lstat().
//
//If you do not know whether a given object is a symlink or not, it's a good
//idea to call Object.SymlinkHeaders() first: If the object turns out not to be
//a symlink, the cache for Object.Headers() has already been populated.
//
//This operation fails with http.StatusNotFound if the object does not exist.
//
//WARNING: This method is not thread-safe. Calling it concurrently on the same
//object results in undefined behavior.
func (o *Object) SymlinkHeaders() (headers ObjectHeaders, target *Object, err error) {
	if o.symlinkHeaders == nil {
		o.symlinkHeaders, err = o.fetchHeaders(&RequestOptions{
			Values: url.Values{"symlink": []string{"get"}},
		})
		if err != nil {
			return ObjectHeaders{}, nil, err
		}
	}

	//is this a symlink?
	targetFullName := o.symlinkHeaders.Get("X-Symlink-Target")
	if targetFullName == "" {
		//not a symlink - the o.symlinkHeaders are just the regular headers
		o.headers = o.symlinkHeaders
		return *o.headers, nil, nil
	}
	fields := strings.SplitN(targetFullName, "/", 2)
	if len(fields) < 2 {
		return ObjectHeaders{}, nil, MalformedHeaderError{
			Key:        "X-Symlink-Target",
			ParseError: fmt.Errorf("expected \"container/object\", got \"%s\"", targetFullName),
		}
	}

	//cross-account symlink?
	accountName := o.symlinkHeaders.Get("X-Symlink-Target-Account")
	targetAccount := o.c.a
	if accountName != "" && accountName != targetAccount.Name() {
		targetAccount = targetAccount.SwitchAccount(accountName)
	}
	target = targetAccount.Container(fields[0]).Object(fields[1])
	return *o.symlinkHeaders, target, nil
}

//URL returns the canonical URL for the object on the server. This is
//particularly useful when the ReadACL on the account or container is set to
//allow anonymous read access.
func (o *Object) URL() (string, error) {
	return Request{
		ContainerName: o.c.name,
		ObjectName:    o.name,
	}.URL(o.c.a.backend, nil)
}

//TempURL is like Object.URL, but includes a token with a limited lifetime (as
//specified by the `expires` argument) that permits anonymous access to this
//object using the given HTTP method. This works only when the tempurl
//middleware is set up on the server, and if the given `key` matches one of the
//tempurl keys for this object's container or account.
//
//For example, if the ReadACL both on the account and container do not permit
//anonymous read access (which is the default behavior):
//
//	var o *schwift.Object
//	...
//	resp, err := http.Get(o.URL())
//	//After this, resp.StatusCode == 401 (Unauthorized)
//	//because anonymous access is forbidden.
//
//	//But if the container or account has a tempurl key...
//	key := "supersecretkey"
//	hdr := NewContainerHeaders()
//	hdr.TempURLKey().Set(key)
//	c := o.Container()
//	err := c.Update(hdr, nil)
//
//	//...we can use it to generate temporary URLs.
//	url := o.TempURL(key, "GET", time.Now().Add(10 * time.Minute))
//	resp, err := http.Get(url)
//	//This time, resp.StatusCode == 200 because the URL includes a token.
//
func (o *Object) TempURL(key, method string, expires time.Time) (string, error) {
	urlStr, err := o.URL()
	if err != nil {
		return "", err
	}
	u, err := url.Parse(urlStr)
	if err != nil {
		return "", err
	}

	payload := fmt.Sprintf("%s\n%d\n%s", method, expires.Unix(), u.Path)
	mac := hmac.New(sha1.New, []byte(key))
	mac.Write([]byte(payload))
	signature := hex.EncodeToString(mac.Sum(nil))

	u.RawQuery = fmt.Sprintf("temp_url_sig=%s&temp_url_expires=%d",
		signature, expires.Unix())
	return u.String(), nil
}