blobstore.go

// Copyright 2011 Google Inc. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.

// Package blobstore provides a client for App Engine's persistent blob
// storage service.
package blobstore // import "google.golang.org/appengine/v2/blobstore"

import (
	"bufio"
	"bytes"
	"context"
	"encoding/base64"
	"fmt"
	"io"
	"io/ioutil"
	"mime"
	"mime/multipart"
	"net/http"
	"net/textproto"
	"net/url"
	"strconv"
	"strings"
	"time"

	"github.com/golang/protobuf/proto"
	"golang.org/x/text/encoding/htmlindex"

	"google.golang.org/appengine/v2"
	"google.golang.org/appengine/v2/datastore"
	"google.golang.org/appengine/v2/internal"

	basepb "google.golang.org/appengine/v2/internal/base"
	blobpb "google.golang.org/appengine/v2/internal/blobstore"
)

const (
	blobInfoKind      = "__BlobInfo__"
	blobFileIndexKind = "__BlobFileIndex__"
	zeroKey           = appengine.BlobKey("")
)

// BlobInfo is the blob metadata that is stored in the datastore.
// Filename may be empty.
type BlobInfo struct {
	BlobKey      appengine.BlobKey
	ContentType  string    `datastore:"content_type"`
	CreationTime time.Time `datastore:"creation"`
	Filename     string    `datastore:"filename"`
	Size         int64     `datastore:"size"`
	MD5          string    `datastore:"md5_hash"`

	// ObjectName is the Google Cloud Storage name for this blob.
	ObjectName string `datastore:"gs_object_name"`
}

// isErrFieldMismatch returns whether err is a datastore.ErrFieldMismatch.
//
// The blobstore stores blob metadata in the datastore. When loading that
// metadata, it may contain fields that we don't care about. datastore.Get will
// return datastore.ErrFieldMismatch in that case, so we ignore that specific
// error.
func isErrFieldMismatch(err error) bool {
	_, ok := err.(*datastore.ErrFieldMismatch)
	return ok
}

// Stat returns the BlobInfo for a provided blobKey. If no blob was found for
// that key, Stat returns datastore.ErrNoSuchEntity.
func Stat(c context.Context, blobKey appengine.BlobKey) (*BlobInfo, error) {
	c, _ = appengine.Namespace(c, "") // Blobstore is always in the empty string namespace
	dskey := datastore.NewKey(c, blobInfoKind, string(blobKey), 0, nil)
	bi := &BlobInfo{
		BlobKey: blobKey,
	}
	if err := datastore.Get(c, dskey, bi); err != nil && !isErrFieldMismatch(err) {
		return nil, err
	}
	return bi, nil
}

// Send sets the headers on response to instruct App Engine to send a blob as
// the response body. This is more efficient than reading and writing it out
// manually and isn't subject to normal response size limits.
func Send(response http.ResponseWriter, blobKey appengine.BlobKey) {
	hdr := response.Header()
	hdr.Set("X-AppEngine-BlobKey", string(blobKey))

	if hdr.Get("Content-Type") == "" {
		// This value is known to dev_appserver to mean automatic.
		// In production this is remapped to the empty value which
		// means automatic.
		hdr.Set("Content-Type", "application/vnd.google.appengine.auto")
	}
}

// UploadURL creates an upload URL for the form that the user will
// fill out, passing the application path to load when the POST of the
// form is completed. These URLs expire and should not be reused. The
// opts parameter may be nil.
func UploadURL(c context.Context, successPath string, opts *UploadURLOptions) (*url.URL, error) {
	req := &blobpb.CreateUploadURLRequest{
		SuccessPath: proto.String(successPath),
	}
	if opts != nil {
		if n := opts.MaxUploadBytes; n != 0 {
			req.MaxUploadSizeBytes = &n
		}
		if n := opts.MaxUploadBytesPerBlob; n != 0 {
			req.MaxUploadSizePerBlobBytes = &n
		}
		if s := opts.StorageBucket; s != "" {
			req.GsBucketName = &s
		}
	}
	res := &blobpb.CreateUploadURLResponse{}
	if err := internal.Call(c, "blobstore", "CreateUploadURL", req, res); err != nil {
		return nil, err
	}
	return url.Parse(*res.Url)
}

// UploadURLOptions are the options to create an upload URL.
type UploadURLOptions struct {
	MaxUploadBytes        int64 // optional
	MaxUploadBytesPerBlob int64 // optional

	// StorageBucket specifies the Google Cloud Storage bucket in which
	// to store the blob.
	// This is required if you use Cloud Storage instead of Blobstore.
	// Your application must have permission to write to the bucket.
	// You may optionally specify a bucket name and path in the format
	// "bucket_name/path", in which case the included path will be the
	// prefix of the uploaded object's name.
	StorageBucket string
}

// Delete deletes a blob.
func Delete(c context.Context, blobKey appengine.BlobKey) error {
	return DeleteMulti(c, []appengine.BlobKey{blobKey})
}

// DeleteMulti deletes multiple blobs.
func DeleteMulti(c context.Context, blobKey []appengine.BlobKey) error {
	s := make([]string, len(blobKey))
	for i, b := range blobKey {
		s[i] = string(b)
	}
	req := &blobpb.DeleteBlobRequest{
		BlobKey: s,
	}
	res := &basepb.VoidProto{}
	if err := internal.Call(c, "blobstore", "DeleteBlob", req, res); err != nil {
		return err
	}
	return nil
}

func errorf(format string, args ...interface{}) error {
	return fmt.Errorf("blobstore: "+format, args...)
}

// ParseUpload parses the synthetic POST request that your app gets from
// App Engine after a user's successful upload of blobs. Given the request,
// ParseUpload returns a map of the blobs received (keyed by HTML form
// element name) and other non-blob POST parameters.
func ParseUpload(req *http.Request) (blobs map[string][]*BlobInfo, other url.Values, err error) {
	_, params, err := mime.ParseMediaType(req.Header.Get("Content-Type"))
	if err != nil {
		return nil, nil, err
	}
	boundary := params["boundary"]
	if boundary == "" {
		return nil, nil, errorf("did not find MIME multipart boundary")
	}

	blobs = make(map[string][]*BlobInfo)
	other = make(url.Values)

	mreader := multipart.NewReader(io.MultiReader(req.Body, strings.NewReader("\r\n\r\n")), boundary)
	for {
		part, perr := mreader.NextPart()
		if perr == io.EOF {
			break
		}
		if perr != nil {
			return nil, nil, errorf("error reading next mime part with boundary %q (len=%d): %v",
				boundary, len(boundary), perr)
		}

		bi := &BlobInfo{}
		ctype, params, err := mime.ParseMediaType(part.Header.Get("Content-Disposition"))
		if err != nil {
			return nil, nil, err
		}
		bi.Filename = params["filename"]
		formKey := params["name"]

		ctype, params, err = mime.ParseMediaType(part.Header.Get("Content-Type"))
		if err != nil {
			return nil, nil, err
		}
		bi.BlobKey = appengine.BlobKey(params["blob-key"])
		charset := params["charset"]

		if ctype != "message/external-body" || bi.BlobKey == "" {
			if formKey != "" {
				slurp, serr := ioutil.ReadAll(part)
				if serr != nil {
					return nil, nil, errorf("error reading %q MIME part", formKey)
				}

				// Handle base64 content transfer encoding. multipart.Part transparently
				// handles quoted-printable, and no special handling is required for
				// 7bit, 8bit, or binary.
				ctype, params, err = mime.ParseMediaType(part.Header.Get("Content-Transfer-Encoding"))
				if err == nil && ctype == "base64" {
					slurp, serr = ioutil.ReadAll(base64.NewDecoder(
						base64.StdEncoding, bytes.NewReader(slurp)))
					if serr != nil {
						return nil, nil, errorf("error %s decoding %q MIME part", ctype, formKey)
					}
				}

				// Handle charset
				if charset != "" {
					encoding, err := htmlindex.Get(charset)
					if err != nil {
						return nil, nil, errorf("error getting decoder for charset %q", charset)
					}

					slurp, err = encoding.NewDecoder().Bytes(slurp)
					if err != nil {
						return nil, nil, errorf("error decoding from charset %q", charset)
					}
				}

				other[formKey] = append(other[formKey], string(slurp))
			}
			continue
		}

		// App Engine sends a MIME header as the body of each MIME part.
		tp := textproto.NewReader(bufio.NewReader(part))
		header, mimeerr := tp.ReadMIMEHeader()
		if mimeerr != nil {
			return nil, nil, mimeerr
		}
		bi.Size, err = strconv.ParseInt(header.Get("Content-Length"), 10, 64)
		if err != nil {
			return nil, nil, err
		}
		bi.ContentType = header.Get("Content-Type")

		// Parse the time from the MIME header like:
		// X-AppEngine-Upload-Creation: 2011-03-15 21:38:34.712136
		createDate := header.Get("X-AppEngine-Upload-Creation")
		if createDate == "" {
			return nil, nil, errorf("expected to find an X-AppEngine-Upload-Creation header")
		}
		bi.CreationTime, err = time.Parse("2006-01-02 15:04:05.000000", createDate)
		if err != nil {
			return nil, nil, errorf("error parsing X-AppEngine-Upload-Creation: %s", err)
		}

		if hdr := header.Get("Content-MD5"); hdr != "" {
			md5, err := base64.URLEncoding.DecodeString(hdr)
			if err != nil {
				return nil, nil, errorf("bad Content-MD5 %q: %v", hdr, err)
			}
			bi.MD5 = string(md5)
		}

		// If the GCS object name was provided, record it.
		bi.ObjectName = header.Get("X-AppEngine-Cloud-Storage-Object")

		blobs[formKey] = append(blobs[formKey], bi)
	}
	return
}

// Reader is a blob reader.
type Reader interface {
	io.Reader
	io.ReaderAt
	io.Seeker
}

// NewReader returns a reader for a blob. It always succeeds; if the blob does
// not exist then an error will be reported upon first read.
func NewReader(c context.Context, blobKey appengine.BlobKey) Reader {
	return openBlob(c, blobKey)
}

// BlobKeyForFile returns a BlobKey for a Google Storage file.
// The filename should be of the form "/gs/bucket_name/object_name".
func BlobKeyForFile(c context.Context, filename string) (appengine.BlobKey, error) {
	req := &blobpb.CreateEncodedGoogleStorageKeyRequest{
		Filename: &filename,
	}
	res := &blobpb.CreateEncodedGoogleStorageKeyResponse{}
	if err := internal.Call(c, "blobstore", "CreateEncodedGoogleStorageKey", req, res); err != nil {
		return "", err
	}
	return appengine.BlobKey(*res.BlobKey), nil
}