datastore_export_job_check_api.go

package main

import (
	"context"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"

	"github.com/gcpug/ds2bq/datastore"
	"github.com/morikuni/failure"
)

type DatastoreExportJobCheckRequest struct {
	DS2BQJobID           string
	DatastoreExportJobID string
}

func HandleDatastoreExportJobCheckAPI(w http.ResponseWriter, r *http.Request) {
	b, err := ioutil.ReadAll(r.Body)
	if err != nil {
		msg := fmt.Sprintf("failed ioutil.Read(request.Body).err=%+v", err)
		log.Println(msg)
		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
		w.WriteHeader(http.StatusBadRequest)
		_, err := w.Write([]byte(msg))
		if err != nil {
			log.Println(err)
		}
		return
	}

	form := &DatastoreExportJobCheckRequest{}
	if err := json.Unmarshal(b, form); err != nil {
		msg := fmt.Sprintf("failed json.Unmarshal(request.Body).err=%+v", err)
		log.Println(msg)
		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
		w.WriteHeader(http.StatusBadRequest)
		_, err := w.Write([]byte(msg))
		if err != nil {
			log.Println(err)
		}
		return
	}

	log.Printf("%s\n", string(b))

	dseJobStore, err := NewDSExportJobStore(r.Context(), DatastoreClient)
	if err != nil {
		msg := fmt.Sprintf("failed NewDSExportJobStore.err=%+v", err)
		log.Println(msg)
		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
		w.WriteHeader(http.StatusInternalServerError)
		_, err := w.Write([]byte(msg))
		if err != nil {
			log.Println(err)
		}
		return
	}

	res, err := datastore.CheckJobStatus(r.Context(), form.DatastoreExportJobID)
	if err != nil {
		msg := fmt.Sprintf("failed datastore.CheckJobStatus.err=%+v", err)
		log.Println(msg)
		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
		w.WriteHeader(http.StatusInternalServerError)
		_, err := w.Write([]byte(msg))
		if err != nil {
			log.Println(err)
		}
		return
	}
	switch res.Status {
	case datastore.Running:
		log.Printf("%s is Running...\n", form.DatastoreExportJobID)

		_, err := dseJobStore.IncrementJobStatusCheckCount(r.Context(), form.DS2BQJobID)
		if err != nil {
			log.Printf("failed DSExportJobStore.IncrementJobStatusCheckCount. DS2BQJobID=%v,err=%v\n", form.DS2BQJobID, err)
			w.WriteHeader(http.StatusInternalServerError)
			return
		}
		w.WriteHeader(http.StatusConflict)
	case datastore.Fail:
		log.Printf("%s is Fail. ErrCode=%v,ErrMessage=%v\n", form.DatastoreExportJobID, res.ErrCode, res.ErrMessage)

		_, err := dseJobStore.FinishExportJob(r.Context(), form.DS2BQJobID, DSExportJobStatusFailed, fmt.Sprintf("Code=%v,MSG=%v,META=%+v", res.ErrCode, res.ErrMessage, res.Metadata))
		if err != nil {
			log.Printf("failed DSExportJobStore.FinishExportJob. DS2BQJobID=%v,err=%v\n", form.DS2BQJobID, err)
			w.WriteHeader(http.StatusInternalServerError)
			return
		}
		w.WriteHeader(http.StatusOK)
	case datastore.Done:
		log.Printf("%s is Done...\n", form.DatastoreExportJobID)

		_, err := dseJobStore.FinishExportJob(r.Context(), form.DS2BQJobID, DSExportJobStatusDone, "")
		if err != nil {
			log.Printf("failed DSExportJobStore.FinishExportJob. DS2BQJobID=%v,err=%v\n", form.DS2BQJobID, err)
			w.WriteHeader(http.StatusInternalServerError)
			return
		}

		if err := InsertBQLoadJobs(r.Context(), form.DS2BQJobID, res.Metadata.OutputURLPrefix, r.Host); err != nil {
			log.Printf("failed InsertBQLoadJobs. DS2BQJobID=%v,err=%v\n", form.DS2BQJobID, err)
			w.WriteHeader(http.StatusInternalServerError)
			return
		}
		w.WriteHeader(http.StatusOK)
	default:
		log.Printf("%v is Unsupported Status\n", res.Status)
		w.WriteHeader(http.StatusInternalServerError)
	}
}

func InsertBQLoadJobs(ctx context.Context, ds2bqJobID string, outputURLPrefix string, host string) error {
	bljs, err := NewBQLoadJobStore(ctx, DatastoreClient)
	if err != nil {
		return failure.Wrap(err, failure.Message("failed NewBQLoadJobStore"))
	}
	q, err := NewBQLoadJobCheckQueue(host, TasksClient)
	if err != nil {
		return failure.Wrap(err, failure.Message("failed NewBQLoadJobCheckQueue"))
	}
	ls := NewBQLoadService(bljs, q)

	if err := ls.InsertBigQueryLoadJob(ctx, ds2bqJobID, outputURLPrefix); err != nil {
		return failure.Wrap(err, failure.Message("failed BQLoadService.InsertBigQueryLoadJob"))
	}

	return nil
}