feat: import dataset from google sheets (#16)

* - prepare graphql mutation and types - prepare the controller * implement load google sheet method * regenerate graphql * restructure the feature in layers * - fix some names - set imported dataset name as sheet name * - add test case * - Use readCloser instead of using files * - fix method name * - remove use cases dependency * - remove token
reearth · Jul 8, 2021 · 2ef7efd · 2ef7efd
1 parent dcb4b08
commit 2ef7efd
Show file tree

Hide file tree

Showing 15 changed files with 383 additions and 16 deletions.
diff --git a/go.mod b/go.mod
@@ -50,6 +50,7 @@ require (
 	golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect
 	golang.org/x/tools v0.1.0
 	gopkg.in/go-playground/colors.v1 v1.2.0
+	gopkg.in/h2non/gock.v1 v1.1.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0
 	gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 // indirect
 )

diff --git a/go.sum b/go.sum
@@ -232,6 +232,8 @@ github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB7
 github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw=
+github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
@@ -310,6 +312,7 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4=
@@ -809,6 +812,8 @@ gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/go-playground/colors.v1 v1.2.0 h1:SPweMUve+ywPrfwao+UvfD5Ah78aOLUkT5RlJiZn52c=
 gopkg.in/go-playground/colors.v1 v1.2.0/go.mod h1:AvbqcMpNXVl5gBrM20jBm3VjjKBbH/kI5UnqjU7lxFI=
+gopkg.in/h2non/gock.v1 v1.1.0 h1:Yy6sSXyTP9wYc6+H7U0NuB1LQ6H2HYmDp2sxFQ8vTEY=
+gopkg.in/h2non/gock.v1 v1.1.0/go.mod h1:n7UGz/ckNChHiK05rDoiC4MYSunEC/lyaUm2WWaDva0=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

diff --git a/internal/adapter/graphql/controller_dataset.go b/internal/adapter/graphql/controller_dataset.go
@@ -78,6 +78,21 @@ func (c *DatasetController) ImportDataset(ctx context.Context, i *ImportDatasetI
 	return &ImportDatasetPayload{DatasetSchema: toDatasetSchema(res)}, nil
 }
 
+func (c *DatasetController) ImportDatasetFromGoogleSheet(ctx context.Context, i *ImportDatasetFromGoogleSheetInput, o *usecase.Operator) (*ImportDatasetPayload, error) {
+	res, err := c.usecase().ImportDatasetFromGoogleSheet(ctx, interfaces.ImportDatasetFromGoogleSheetParam{
+		Token:     i.AccessToken,
+		FileID:    i.FileID,
+		SheetName: i.SheetName,
+		SceneId:   id.SceneID(i.SceneID),
+		SchemaId:  id.DatasetSchemaIDFromRefID(i.DatasetSchemaID),
+	}, o)
+	if err != nil {
+		return nil, err
+	}
+
+	return &ImportDatasetPayload{DatasetSchema: toDatasetSchema(res)}, nil
+}
+
 func (c *DatasetController) GraphFetchSchema(ctx context.Context, i id.ID, depth int, operator *usecase.Operator) ([]*DatasetSchema, []error) {
 	res, err := c.usecase().GraphFetchSchema(ctx, id.DatasetSchemaID(i), depth, operator)
 	if err != nil {

diff --git a/internal/adapter/graphql/models_gen.go b/internal/adapter/graphql/models_gen.go
diff --git a/internal/app/repo.go b/internal/app/repo.go
@@ -6,6 +6,11 @@ import (
 	"time"
 
 	"github.com/reearth/reearth-backend/internal/infrastructure/github"
+	"github.com/reearth/reearth-backend/internal/infrastructure/google"
+
+	"go.mongodb.org/mongo-driver/mongo"
+	"go.mongodb.org/mongo-driver/mongo/options"
+	mongotrace "go.opentelemetry.io/contrib/instrumentation/go.mongodb.org/mongo-driver"
 
 	"github.com/reearth/reearth-backend/internal/infrastructure/adapter"
 	"github.com/reearth/reearth-backend/internal/infrastructure/auth0"
@@ -15,9 +20,6 @@ import (
 	"github.com/reearth/reearth-backend/internal/usecase/gateway"
 	"github.com/reearth/reearth-backend/internal/usecase/repo"
 	"github.com/reearth/reearth-backend/pkg/log"
-	"go.mongodb.org/mongo-driver/mongo"
-	"go.mongodb.org/mongo-driver/mongo/options"
-	mongotrace "go.opentelemetry.io/contrib/instrumentation/go.mongodb.org/mongo-driver"
 )
 
 func initReposAndGateways(ctx context.Context, conf *Config, debug bool) (*repo.Container, *gateway.Container) {
@@ -77,6 +79,9 @@ func initReposAndGateways(ctx context.Context, conf *Config, debug bool) (*repo.
 	// github
 	gateways.PluginRegistry = github.NewPluginRegistry()
 
+	// google
+	gateways.Google = google.NewGoogle()
+
 	// release lock of all scenes
 	if err := repos.SceneLock.ReleaseAllLock(context.Background()); err != nil {
 		log.Fatalln(fmt.Sprintf("repo initialization error: %+v", err))

diff --git a/internal/graphql/generated.go b/internal/graphql/generated.go
diff --git a/internal/graphql/resolver_mutation.go b/internal/graphql/resolver_mutation.go
@@ -425,3 +425,10 @@ func (r *mutationResolver) ImportDataset(ctx context.Context, input graphql1.Imp
 
 	return r.config.Controllers.DatasetController.ImportDataset(ctx, &input, getOperator(ctx))
 }
+
+func (r *mutationResolver) ImportDatasetFromGoogleSheet(ctx context.Context, input graphql1.ImportDatasetFromGoogleSheetInput) (*graphql1.ImportDatasetPayload, error) {
+	exit := trace(ctx)
+	defer exit()
+
+	return r.config.Controllers.DatasetController.ImportDatasetFromGoogleSheet(ctx, &input, getOperator(ctx))
+}
diff --git a/internal/infrastructure/google/fetch.go b/internal/infrastructure/google/fetch.go
@@ -0,0 +1,26 @@
+package google
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+)
+
+func fetchCSV(token string, fileId string, sheetName string) (*io.ReadCloser, error) {
+	url := fmt.Sprintf("https://docs.google.com/spreadsheets/d/%s/gviz/tq?tqx=out:csv&sheet=%s", fileId, sheetName)
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	req.Header.Set("Authorization", "Bearer "+token)
+	res, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	if res.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("StatusCode=%d", res.StatusCode)
+	}
+
+	return &res.Body, nil
+}