add image upload& s3 upload api (#7515)

Co-authored-by: TangBin <tangbin@yunion.cn>

pkg/apigateway/handler/imageutils.go

@@ -0,0 +1,129 @@
+package handler
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"mime/multipart"
+	"net/http"
+	"strconv"
+	"time"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+
+	"yunion.io/x/onecloud/pkg/appsrv"
+	"yunion.io/x/onecloud/pkg/httperrors"
+	"yunion.io/x/onecloud/pkg/mcclient/auth"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+)
+
+func readImageForm(r *multipart.Reader) (map[string]string, *multipart.Part, error) {
+	params := make(map[string]string)
+
+	maxValueBytes := int64(10 << 20)
+	for {
+		p, err := r.NextPart()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return nil, nil, err
+		}
+
+		name := p.FormName()
+		if name == "" {
+			continue
+		}
+		filename := p.FileName()
+
+		var b bytes.Buffer
+
+		_, hasContentTypeHeader := p.Header["Content-Type"]
+		if !hasContentTypeHeader && filename == "" {
+			// value, store as string in memory
+			n, err := io.CopyN(&b, p, maxValueBytes+1)
+			if err != nil && err != io.EOF {
+				return nil, nil, err
+			}
+			maxValueBytes -= n
+			if maxValueBytes < 0 {
+				return nil, nil, multipart.ErrMessageTooLarge
+			}
+			params[name] = b.String()
+			continue
+		}
+
+		if name == "image" || name == "file" {
+			return params, p, nil
+		} else {
+			return nil, nil, fmt.Errorf("no file uploaded")
+		}
+	}
+	return nil, nil, fmt.Errorf("empty form")
+}
+
+func imageUploadHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
+	reader, e := r.MultipartReader()
+	if e != nil {
+		httperrors.InvalidInputError(w, "无效的表单")
+		return
+	}
+
+	p, f, e := readImageForm(reader)
+	if e != nil {
+		httperrors.InvalidInputError(w, "无效的表单")
+		return
+	}
+
+	params := jsonutils.NewDict()
+
+	name, ok := p["name"]
+	if !ok {
+		httperrors.InvalidInputError(w, "缺少镜像名称")
+		return
+	}
+	params.Add(jsonutils.NewString(name), "name")
+
+	_imageSize, ok := p["image_size"]
+	if !ok {
+		httperrors.InvalidInputError(w, "缺少文件信息")
+		return
+	}
+	imageSize, e := strconv.ParseInt(_imageSize, 10, 64)
+	if e != nil {
+		httperrors.InvalidInputError(w, "文件信息错误")
+		return
+	}
+
+	// add all other params
+	for k, v := range p {
+		if k == "name" || k == "image_size" {
+			continue
+		}
+		params.Add(jsonutils.NewString(v), k)
+	}
+
+	token := AppContextToken(ctx)
+	s := auth.GetSession(ctx, token, FetchRegion(r), "")
+
+	res, e := modules.Images.Upload(s, params, f, imageSize)
+	if e != nil {
+		httperrors.GeneralServerError(w, e)
+		return
+	} else {
+		appsrv.SendJSON(w, res)
+	}
+}
+
+func uploadHandlerInfo(method, prefix string, handler func(context.Context, http.ResponseWriter, *http.Request)) *appsrv.SHandlerInfo {
+	log.Debugf("%s - %s", method, prefix)
+	hi := appsrv.SHandlerInfo{}
+	hi.SetMethod(method)
+	hi.SetPath(prefix)
+	hi.SetHandler(handler)
+	hi.SetProcessTimeout(6 * time.Hour)
+	hi.SetWorkerManager(GetUploaderWorker())
+	return &hi
+}

pkg/apigateway/handler/misc.go

@@ -20,6 +20,7 @@ import (
 	"encoding/csv"
 	"fmt"
 	"net/http"
+	"strconv"
 	"strings"
 	"time"
 
@@ -90,6 +91,10 @@ func (h *MiscHandler) Bind(app *appsrv.Application) {
 	app.AddHandler3(uploader)
 	app.AddHandler(GET, prefix+"downloads/<template_id>", FetchAuthToken(h.getDownloadsHandler))
 	app.AddHandler(POST, prefix+"piuploads", FetchAuthToken(h.postPIUploads)) // itsm process instances upload api
+	imageUploader := uploadHandlerInfo("POST", prefix+"/imageutils/upload", FetchAuthToken(imageUploadHandler))
+	app.AddHandler3(imageUploader)
+	s3upload := uploadHandlerInfo(POST, prefix+"s3uploads", FetchAuthToken(h.postS3UploadHandler))
+	app.AddHandler3(s3upload)
 }
 
 func UploadHandlerInfo(method, prefix string, handler func(context.Context, http.ResponseWriter, *http.Request)) *appsrv.SHandlerInfo {
@@ -509,6 +514,61 @@ func (mh *MiscHandler) postPIUploads(ctx context.Context, w http.ResponseWriter,
 	appsrv.SendJSON(w, resp)
 }
 
+func (mh *MiscHandler) postS3UploadHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
+	reader, e := r.MultipartReader()
+	if e != nil {
+		log.Debugf("postS3UploadHandler.MultipartReader %s", e)
+		httperrors.InvalidInputError(w, "invalid form")
+		return
+	}
+
+	p, f, e := readImageForm(reader)
+	if e != nil {
+		log.Debugf("postS3UploadHandler.readImageForm %s", e)
+		httperrors.InvalidInputError(w, "invalid form")
+		return
+	}
+
+	bucket_id, ok := p["bucket_id"]
+	if !ok {
+		httperrors.MissingParameterError(w, "bucket_id")
+		return
+	}
+
+	key, ok := p["key"]
+	if !ok {
+		httperrors.MissingParameterError(w, "key")
+		return
+	}
+
+	_content_length, ok := p["content_length"]
+	if !ok {
+		httperrors.MissingParameterError(w, "content_length")
+		return
+	}
+
+	content_length, e := strconv.ParseInt(_content_length, 10, 64)
+	if e != nil {
+		httperrors.InvalidInputError(w, "invalid content_length %s", _content_length)
+		return
+	}
+
+	storage_class, _ := p["storage_class"]
+	acl, _ := p["acl"]
+
+	token := AppContextToken(ctx)
+	s := auth.GetSession(ctx, token, FetchRegion(r), "")
+
+	meta := http.Header{}
+	meta.Set("Content-Type", "application/octet-stream")
+	e = modules.Buckets.Upload(s, bucket_id, key, f, content_length, storage_class, acl, meta)
+	if e != nil {
+		httperrors.GeneralServerError(w, e)
+		return
+	}
+	appsrv.SendJSON(w, jsonutils.NewDict())
+}
+
 func writeCsv(records [][]string) (bytes.Buffer, error) {
 	var content bytes.Buffer
 	content.WriteString("\xEF\xBB\xBF") // 写入UTF-8 BOM, 防止office打开后中文乱码