From 2ef7efdd917cd95aa43bec3c0d2b24a0bbeccd22 Mon Sep 17 00:00:00 2001 From: yk-eukarya <81808708+yk-eukarya@users.noreply.github.com> Date: Thu, 8 Jul 2021 19:03:08 +0300 Subject: [PATCH] feat: import dataset from google sheets (#16) * - prepare graphql mutation and types - prepare the controller * implement load google sheet method * regenerate graphql * restructure the feature in layers * - fix some names - set imported dataset name as sheet name * - add test case * - Use readCloser instead of using files * - fix method name * - remove use cases dependency * - remove token --- go.mod | 1 + go.sum | 5 + .../adapter/graphql/controller_dataset.go | 15 ++ internal/adapter/graphql/models_gen.go | 8 ++ internal/app/repo.go | 11 +- internal/graphql/generated.go | 136 ++++++++++++++++++ internal/graphql/resolver_mutation.go | 7 + internal/infrastructure/google/fetch.go | 26 ++++ internal/infrastructure/google/fetch_test.go | 90 ++++++++++++ internal/infrastructure/google/google.go | 18 +++ internal/usecase/gateway/container.go | 1 + internal/usecase/gateway/google.go | 9 ++ internal/usecase/interactor/dataset.go | 54 +++++-- internal/usecase/interfaces/dataset.go | 9 ++ schema.graphql | 9 ++ 15 files changed, 383 insertions(+), 16 deletions(-) create mode 100644 internal/infrastructure/google/fetch.go create mode 100644 internal/infrastructure/google/fetch_test.go create mode 100644 internal/infrastructure/google/google.go create mode 100644 internal/usecase/gateway/google.go diff --git a/go.mod b/go.mod index 57c096f8..553d5242 100644 --- a/go.mod +++ b/go.mod @@ -50,6 +50,7 @@ require ( golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect golang.org/x/tools v0.1.0 gopkg.in/go-playground/colors.v1 v1.2.0 + gopkg.in/h2non/gock.v1 v1.1.0 // indirect gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 // indirect ) diff --git a/go.sum b/go.sum index 249bd033..b4dd9e75 100644 --- a/go.sum +++ b/go.sum @@ -232,6 +232,8 @@ github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB7 github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw= +github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= @@ -310,6 +312,7 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= @@ -809,6 +812,8 @@ gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/go-playground/colors.v1 v1.2.0 h1:SPweMUve+ywPrfwao+UvfD5Ah78aOLUkT5RlJiZn52c= gopkg.in/go-playground/colors.v1 v1.2.0/go.mod h1:AvbqcMpNXVl5gBrM20jBm3VjjKBbH/kI5UnqjU7lxFI= +gopkg.in/h2non/gock.v1 v1.1.0 h1:Yy6sSXyTP9wYc6+H7U0NuB1LQ6H2HYmDp2sxFQ8vTEY= +gopkg.in/h2non/gock.v1 v1.1.0/go.mod h1:n7UGz/ckNChHiK05rDoiC4MYSunEC/lyaUm2WWaDva0= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/internal/adapter/graphql/controller_dataset.go b/internal/adapter/graphql/controller_dataset.go index df729bab..dcf974d3 100644 --- a/internal/adapter/graphql/controller_dataset.go +++ b/internal/adapter/graphql/controller_dataset.go @@ -78,6 +78,21 @@ func (c *DatasetController) ImportDataset(ctx context.Context, i *ImportDatasetI return &ImportDatasetPayload{DatasetSchema: toDatasetSchema(res)}, nil } +func (c *DatasetController) ImportDatasetFromGoogleSheet(ctx context.Context, i *ImportDatasetFromGoogleSheetInput, o *usecase.Operator) (*ImportDatasetPayload, error) { + res, err := c.usecase().ImportDatasetFromGoogleSheet(ctx, interfaces.ImportDatasetFromGoogleSheetParam{ + Token: i.AccessToken, + FileID: i.FileID, + SheetName: i.SheetName, + SceneId: id.SceneID(i.SceneID), + SchemaId: id.DatasetSchemaIDFromRefID(i.DatasetSchemaID), + }, o) + if err != nil { + return nil, err + } + + return &ImportDatasetPayload{DatasetSchema: toDatasetSchema(res)}, nil +} + func (c *DatasetController) GraphFetchSchema(ctx context.Context, i id.ID, depth int, operator *usecase.Operator) ([]*DatasetSchema, []error) { res, err := c.usecase().GraphFetchSchema(ctx, id.DatasetSchemaID(i), depth, operator) if err != nil { diff --git a/internal/adapter/graphql/models_gen.go b/internal/adapter/graphql/models_gen.go index 1fc6e0b4..f3319715 100644 --- a/internal/adapter/graphql/models_gen.go +++ b/internal/adapter/graphql/models_gen.go @@ -316,6 +316,14 @@ type DeleteTeamPayload struct { TeamID id.ID `json:"teamId"` } +type ImportDatasetFromGoogleSheetInput struct { + AccessToken string `json:"accessToken"` + FileID string `json:"fileId"` + SheetName string `json:"sheetName"` + SceneID id.ID `json:"sceneId"` + DatasetSchemaID *id.ID `json:"datasetSchemaId"` +} + type ImportDatasetInput struct { File graphql.Upload `json:"file"` SceneID id.ID `json:"sceneId"` diff --git a/internal/app/repo.go b/internal/app/repo.go index ea1b4897..7050f275 100644 --- a/internal/app/repo.go +++ b/internal/app/repo.go @@ -6,6 +6,11 @@ import ( "time" "github.com/reearth/reearth-backend/internal/infrastructure/github" + "github.com/reearth/reearth-backend/internal/infrastructure/google" + + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + mongotrace "go.opentelemetry.io/contrib/instrumentation/go.mongodb.org/mongo-driver" "github.com/reearth/reearth-backend/internal/infrastructure/adapter" "github.com/reearth/reearth-backend/internal/infrastructure/auth0" @@ -15,9 +20,6 @@ import ( "github.com/reearth/reearth-backend/internal/usecase/gateway" "github.com/reearth/reearth-backend/internal/usecase/repo" "github.com/reearth/reearth-backend/pkg/log" - "go.mongodb.org/mongo-driver/mongo" - "go.mongodb.org/mongo-driver/mongo/options" - mongotrace "go.opentelemetry.io/contrib/instrumentation/go.mongodb.org/mongo-driver" ) func initReposAndGateways(ctx context.Context, conf *Config, debug bool) (*repo.Container, *gateway.Container) { @@ -77,6 +79,9 @@ func initReposAndGateways(ctx context.Context, conf *Config, debug bool) (*repo. // github gateways.PluginRegistry = github.NewPluginRegistry() + // google + gateways.Google = google.NewGoogle() + // release lock of all scenes if err := repos.SceneLock.ReleaseAllLock(context.Background()); err != nil { log.Fatalln(fmt.Sprintf("repo initialization error: %+v", err)) diff --git a/internal/graphql/generated.go b/internal/graphql/generated.go index 400126b9..dbba8a9d 100644 --- a/internal/graphql/generated.go +++ b/internal/graphql/generated.go @@ -441,6 +441,7 @@ type ComplexityRoot struct { DeleteProject func(childComplexity int, input graphql1.DeleteProjectInput) int DeleteTeam func(childComplexity int, input graphql1.DeleteTeamInput) int ImportDataset func(childComplexity int, input graphql1.ImportDatasetInput) int + ImportDatasetFromGoogleSheet func(childComplexity int, input graphql1.ImportDatasetFromGoogleSheetInput) int ImportLayer func(childComplexity int, input graphql1.ImportLayerInput) int InstallPlugin func(childComplexity int, input graphql1.InstallPluginInput) int LinkDatasetToPropertyValue func(childComplexity int, input graphql1.LinkDatasetToPropertyValueInput) int @@ -1004,6 +1005,7 @@ type MutationResolver interface { AddDynamicDataset(ctx context.Context, input graphql1.AddDynamicDatasetInput) (*graphql1.AddDynamicDatasetPayload, error) RemoveDatasetSchema(ctx context.Context, input graphql1.RemoveDatasetSchemaInput) (*graphql1.RemoveDatasetSchemaPayload, error) ImportDataset(ctx context.Context, input graphql1.ImportDatasetInput) (*graphql1.ImportDatasetPayload, error) + ImportDatasetFromGoogleSheet(ctx context.Context, input graphql1.ImportDatasetFromGoogleSheetInput) (*graphql1.ImportDatasetPayload, error) AddDatasetSchema(ctx context.Context, input graphql1.AddDatasetSchemaInput) (*graphql1.AddDatasetSchemaPayload, error) UpdatePropertyValue(ctx context.Context, input graphql1.UpdatePropertyValueInput) (*graphql1.PropertyFieldPayload, error) UpdatePropertyValueLatLng(ctx context.Context, input graphql1.UpdatePropertyValueLatLngInput) (*graphql1.PropertyFieldPayload, error) @@ -2814,6 +2816,18 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in return e.complexity.Mutation.ImportDataset(childComplexity, args["input"].(graphql1.ImportDatasetInput)), true + case "Mutation.importDatasetFromGoogleSheet": + if e.complexity.Mutation.ImportDatasetFromGoogleSheet == nil { + break + } + + args, err := ec.field_Mutation_importDatasetFromGoogleSheet_args(context.TODO(), rawArgs) + if err != nil { + return 0, false + } + + return e.complexity.Mutation.ImportDatasetFromGoogleSheet(childComplexity, args["input"].(graphql1.ImportDatasetFromGoogleSheetInput)), true + case "Mutation.importLayer": if e.complexity.Mutation.ImportLayer == nil { break @@ -6388,6 +6402,14 @@ input ImportDatasetInput { datasetSchemaId: ID } +input ImportDatasetFromGoogleSheetInput { + accessToken: String! + fileId: String! + sheetName: String! + sceneId: ID! + datasetSchemaId: ID +} + input AddDatasetSchemaInput { sceneId: ID! name: String! @@ -6752,6 +6774,7 @@ type Mutation { input: RemoveDatasetSchemaInput! ): RemoveDatasetSchemaPayload importDataset(input: ImportDatasetInput!): ImportDatasetPayload + importDatasetFromGoogleSheet(input: ImportDatasetFromGoogleSheetInput!): ImportDatasetPayload addDatasetSchema(input: AddDatasetSchemaInput!): AddDatasetSchemaPayload # Property @@ -7102,6 +7125,21 @@ func (ec *executionContext) field_Mutation_deleteTeam_args(ctx context.Context, return args, nil } +func (ec *executionContext) field_Mutation_importDatasetFromGoogleSheet_args(ctx context.Context, rawArgs map[string]interface{}) (map[string]interface{}, error) { + var err error + args := map[string]interface{}{} + var arg0 graphql1.ImportDatasetFromGoogleSheetInput + if tmp, ok := rawArgs["input"]; ok { + ctx := graphql.WithPathContext(ctx, graphql.NewPathWithField("input")) + arg0, err = ec.unmarshalNImportDatasetFromGoogleSheetInput2githubᚗcomᚋreearthᚋreearthᚑbackendᚋinternalᚋadapterᚋgraphqlᚐImportDatasetFromGoogleSheetInput(ctx, tmp) + if err != nil { + return nil, err + } + } + args["input"] = arg0 + return args, nil +} + func (ec *executionContext) field_Mutation_importDataset_args(ctx context.Context, rawArgs map[string]interface{}) (map[string]interface{}, error) { var err error args := map[string]interface{}{} @@ -16568,6 +16606,45 @@ func (ec *executionContext) _Mutation_importDataset(ctx context.Context, field g return ec.marshalOImportDatasetPayload2ᚖgithubᚗcomᚋreearthᚋreearthᚑbackendᚋinternalᚋadapterᚋgraphqlᚐImportDatasetPayload(ctx, field.Selections, res) } +func (ec *executionContext) _Mutation_importDatasetFromGoogleSheet(ctx context.Context, field graphql.CollectedField) (ret graphql.Marshaler) { + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + fc := &graphql.FieldContext{ + Object: "Mutation", + Field: field, + Args: nil, + IsMethod: true, + IsResolver: true, + } + + ctx = graphql.WithFieldContext(ctx, fc) + rawArgs := field.ArgumentMap(ec.Variables) + args, err := ec.field_Mutation_importDatasetFromGoogleSheet_args(ctx, rawArgs) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + fc.Args = args + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { + ctx = rctx // use context from middleware stack in children + return ec.resolvers.Mutation().ImportDatasetFromGoogleSheet(rctx, args["input"].(graphql1.ImportDatasetFromGoogleSheetInput)) + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + return graphql.Null + } + res := resTmp.(*graphql1.ImportDatasetPayload) + fc.Result = res + return ec.marshalOImportDatasetPayload2ᚖgithubᚗcomᚋreearthᚋreearthᚑbackendᚋinternalᚋadapterᚋgraphqlᚐImportDatasetPayload(ctx, field.Selections, res) +} + func (ec *executionContext) _Mutation_addDatasetSchema(ctx context.Context, field graphql.CollectedField) (ret graphql.Marshaler) { defer func() { if r := recover(); r != nil { @@ -28545,6 +28622,58 @@ func (ec *executionContext) unmarshalInputDeleteTeamInput(ctx context.Context, o return it, nil } +func (ec *executionContext) unmarshalInputImportDatasetFromGoogleSheetInput(ctx context.Context, obj interface{}) (graphql1.ImportDatasetFromGoogleSheetInput, error) { + var it graphql1.ImportDatasetFromGoogleSheetInput + var asMap = obj.(map[string]interface{}) + + for k, v := range asMap { + switch k { + case "accessToken": + var err error + + ctx := graphql.WithPathContext(ctx, graphql.NewPathWithField("accessToken")) + it.AccessToken, err = ec.unmarshalNString2string(ctx, v) + if err != nil { + return it, err + } + case "fileId": + var err error + + ctx := graphql.WithPathContext(ctx, graphql.NewPathWithField("fileId")) + it.FileID, err = ec.unmarshalNString2string(ctx, v) + if err != nil { + return it, err + } + case "sheetName": + var err error + + ctx := graphql.WithPathContext(ctx, graphql.NewPathWithField("sheetName")) + it.SheetName, err = ec.unmarshalNString2string(ctx, v) + if err != nil { + return it, err + } + case "sceneId": + var err error + + ctx := graphql.WithPathContext(ctx, graphql.NewPathWithField("sceneId")) + it.SceneID, err = ec.unmarshalNID2githubᚗcomᚋreearthᚋreearthᚑbackendᚋpkgᚋidᚐID(ctx, v) + if err != nil { + return it, err + } + case "datasetSchemaId": + var err error + + ctx := graphql.WithPathContext(ctx, graphql.NewPathWithField("datasetSchemaId")) + it.DatasetSchemaID, err = ec.unmarshalOID2ᚖgithubᚗcomᚋreearthᚋreearthᚑbackendᚋpkgᚋidᚐID(ctx, v) + if err != nil { + return it, err + } + } + } + + return it, nil +} + func (ec *executionContext) unmarshalInputImportDatasetInput(ctx context.Context, obj interface{}) (graphql1.ImportDatasetInput, error) { var it graphql1.ImportDatasetInput var asMap = obj.(map[string]interface{}) @@ -32695,6 +32824,8 @@ func (ec *executionContext) _Mutation(ctx context.Context, sel ast.SelectionSet) out.Values[i] = ec._Mutation_removeDatasetSchema(ctx, field) case "importDataset": out.Values[i] = ec._Mutation_importDataset(ctx, field) + case "importDatasetFromGoogleSheet": + out.Values[i] = ec._Mutation_importDatasetFromGoogleSheet(ctx, field) case "addDatasetSchema": out.Values[i] = ec._Mutation_addDatasetSchema(ctx, field) case "updatePropertyValue": @@ -36471,6 +36602,11 @@ func (ec *executionContext) marshalNID2ᚖgithubᚗcomᚋreearthᚋreearthᚑbac return res } +func (ec *executionContext) unmarshalNImportDatasetFromGoogleSheetInput2githubᚗcomᚋreearthᚋreearthᚑbackendᚋinternalᚋadapterᚋgraphqlᚐImportDatasetFromGoogleSheetInput(ctx context.Context, v interface{}) (graphql1.ImportDatasetFromGoogleSheetInput, error) { + res, err := ec.unmarshalInputImportDatasetFromGoogleSheetInput(ctx, v) + return res, graphql.ErrorOnPath(ctx, err) +} + func (ec *executionContext) unmarshalNImportDatasetInput2githubᚗcomᚋreearthᚋreearthᚑbackendᚋinternalᚋadapterᚋgraphqlᚐImportDatasetInput(ctx context.Context, v interface{}) (graphql1.ImportDatasetInput, error) { res, err := ec.unmarshalInputImportDatasetInput(ctx, v) return res, graphql.ErrorOnPath(ctx, err) diff --git a/internal/graphql/resolver_mutation.go b/internal/graphql/resolver_mutation.go index b10d3658..788969ca 100644 --- a/internal/graphql/resolver_mutation.go +++ b/internal/graphql/resolver_mutation.go @@ -425,3 +425,10 @@ func (r *mutationResolver) ImportDataset(ctx context.Context, input graphql1.Imp return r.config.Controllers.DatasetController.ImportDataset(ctx, &input, getOperator(ctx)) } + +func (r *mutationResolver) ImportDatasetFromGoogleSheet(ctx context.Context, input graphql1.ImportDatasetFromGoogleSheetInput) (*graphql1.ImportDatasetPayload, error) { + exit := trace(ctx) + defer exit() + + return r.config.Controllers.DatasetController.ImportDatasetFromGoogleSheet(ctx, &input, getOperator(ctx)) +} diff --git a/internal/infrastructure/google/fetch.go b/internal/infrastructure/google/fetch.go new file mode 100644 index 00000000..9ea6af9b --- /dev/null +++ b/internal/infrastructure/google/fetch.go @@ -0,0 +1,26 @@ +package google + +import ( + "fmt" + "io" + "net/http" +) + +func fetchCSV(token string, fileId string, sheetName string) (*io.ReadCloser, error) { + url := fmt.Sprintf("https://docs.google.com/spreadsheets/d/%s/gviz/tq?tqx=out:csv&sheet=%s", fileId, sheetName) + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, err + } + + req.Header.Set("Authorization", "Bearer "+token) + res, err := http.DefaultClient.Do(req) + if err != nil { + return nil, err + } + if res.StatusCode != http.StatusOK { + return nil, fmt.Errorf("StatusCode=%d", res.StatusCode) + } + + return &res.Body, nil +} diff --git a/internal/infrastructure/google/fetch_test.go b/internal/infrastructure/google/fetch_test.go new file mode 100644 index 00000000..b7e8d2c4 --- /dev/null +++ b/internal/infrastructure/google/fetch_test.go @@ -0,0 +1,90 @@ +package google + +import ( + "net/http" + "testing" + + "github.com/reearth/reearth-backend/pkg/file" + "github.com/stretchr/testify/assert" + "gopkg.in/h2non/gock.v1" +) + +func Test_fetchCSV(t *testing.T) { + t.Cleanup(func() { + gock.EnableNetworking() + gock.OffAll() + }) + + gock.DisableNetworking() + + type args struct { + token string + fileId string + sheetName string + } + tests := []struct { + name string + setup func() + args args + want *file.File + wantErr bool + }{ + { + name: "Invalid Token", + setup: func() { + gock.New("https://docs.google.com"). + Get("/spreadsheets/d/(.*)/gviz/tq"). + PathParam("d", "1bXBDUrOgYWdHzScMiLNHRUsmNC9SUV4VFOvpqrx0Yok"). + MatchParams(map[string]string{ + "tqx": "out:csv", + "sheet": "Dataset1", + }). + Reply(http.StatusUnauthorized) + }, + args: args{ + token: "xxxx", + fileId: "1bXBDUrOgYWdHzScMiLNHRUsmNC9SUV4VFOvpqrxxxxx", + sheetName: "Dataset1", + }, + wantErr: true, + }, + { + name: "Working scenario", + setup: func() { + gock.New("https://docs.google.com"). + Get("/spreadsheets/d/(.*)/gviz/tq"). + PathParam("d", "1bXBDUrOgYWdHzScMiLNHRUsmNC9SUV4VFOvpqrxxxxx"). + MatchParams(map[string]string{ + "tqx": "out:csv", + "sheet": "Dataset1", + }). + Reply(http.StatusOK). + BodyString("lat,lng,hieght\n30,35,300\n30.1,35,400") + }, + args: args{ + token: "xxxx", + fileId: "1bXBDUrOgYWdHzScMiLNHRUsmNC9SUV4VFOvpqrxxxxx", + sheetName: "Dataset1", + }, + wantErr: false, + }, + } + for _, tt := range tests { + tt := tt + + t.Run(tt.name, func(t *testing.T) { + tt.setup() + + got, err := fetchCSV(tt.args.token, tt.args.fileId, tt.args.sheetName) + if (err != nil) != tt.wantErr { + t.Errorf("fetchCSV() error = %v, wantErr %v", err, tt.wantErr) + return + } + if tt.wantErr { + assert.Nil(t, got) + return + } + assert.NotNil(t, got) + }) + } +} diff --git a/internal/infrastructure/google/google.go b/internal/infrastructure/google/google.go new file mode 100644 index 00000000..7810b278 --- /dev/null +++ b/internal/infrastructure/google/google.go @@ -0,0 +1,18 @@ +package google + +import ( + "io" + + "github.com/reearth/reearth-backend/internal/usecase/gateway" +) + +type google struct { +} + +func NewGoogle() gateway.Google { + return &google{} +} + +func (g google) FetchCSV(token string, fileId string, sheetName string) (*io.ReadCloser, error) { + return fetchCSV(token, fileId, sheetName) +} diff --git a/internal/usecase/gateway/container.go b/internal/usecase/gateway/container.go index 5f730d7e..149e422b 100644 --- a/internal/usecase/gateway/container.go +++ b/internal/usecase/gateway/container.go @@ -7,4 +7,5 @@ type Container struct { DataSource DataSource PluginRegistry PluginRegistry File File + Google Google } diff --git a/internal/usecase/gateway/google.go b/internal/usecase/gateway/google.go new file mode 100644 index 00000000..26655fba --- /dev/null +++ b/internal/usecase/gateway/google.go @@ -0,0 +1,9 @@ +package gateway + +import ( + "io" +) + +type Google interface { + FetchCSV(token string, fileId string, sheetName string) (*io.ReadCloser, error) +} diff --git a/internal/usecase/interactor/dataset.go b/internal/usecase/interactor/dataset.go index a1a0791b..7590ae9c 100644 --- a/internal/usecase/interactor/dataset.go +++ b/internal/usecase/interactor/dataset.go @@ -3,10 +3,12 @@ package interactor import ( "context" "errors" + "io" "strings" "github.com/reearth/reearth-backend/internal/usecase/gateway" "github.com/reearth/reearth-backend/internal/usecase/interfaces" + "github.com/reearth/reearth-backend/pkg/log" "github.com/reearth/reearth-backend/internal/usecase" "github.com/reearth/reearth-backend/internal/usecase/repo" @@ -34,6 +36,7 @@ type Dataset struct { transaction repo.Transaction datasource gateway.DataSource file gateway.File + google gateway.Google } func NewDataset(r *repo.Container, gr *gateway.Container) interfaces.Dataset { @@ -48,6 +51,7 @@ func NewDataset(r *repo.Container, gr *gateway.Container) interfaces.Dataset { transaction: r.Transaction, datasource: gr.DataSource, file: gr.File, + google: gr.Google, } } @@ -185,6 +189,34 @@ func (i *Dataset) ImportDataset(ctx context.Context, inp interfaces.ImportDatase return nil, interfaces.ErrFileNotIncluded } + separator := ',' + if strings.HasSuffix(inp.File.Name, ".tsv") { + separator = '\t' + } + + return i.importDataset(ctx, inp.File.Content, inp.File.Name, separator, inp.SceneId, inp.SchemaId) +} + +func (i *Dataset) ImportDatasetFromGoogleSheet(ctx context.Context, inp interfaces.ImportDatasetFromGoogleSheetParam, operator *usecase.Operator) (_ *dataset.Schema, err error) { + if err := i.CanWriteScene(ctx, inp.SceneId, operator); err != nil { + return nil, err + } + + csvFile, err := i.google.FetchCSV(inp.Token, inp.FileID, inp.SheetName) + if err != nil { + return nil, err + } + defer func() { + err = (*csvFile).Close() + if err != nil { + log.Fatal(err) + } + }() + + return i.importDataset(ctx, *csvFile, inp.SheetName, ',', inp.SceneId, inp.SchemaId) +} + +func (i *Dataset) importDataset(ctx context.Context, content io.Reader, name string, separator rune, sceneId id.SceneID, schemaId *id.DatasetSchemaID) (_ *dataset.Schema, err error) { tx, err := i.transaction.Begin() if err != nil { return @@ -195,20 +227,16 @@ func (i *Dataset) ImportDataset(ctx context.Context, inp interfaces.ImportDatase } }() - seperator := ',' - if strings.HasSuffix(inp.File.Name, ".tsv") { - seperator = '\t' - } - scenes := []id.SceneID{inp.SceneId} - csv := dataset.NewCSVParser(inp.File.Content, inp.File.Name, seperator) + scenes := []id.SceneID{sceneId} + csv := dataset.NewCSVParser(content, name, separator) err = csv.Init() if err != nil { return nil, err } // replacment mode - if inp.SchemaId != nil { - dss, err := i.datasetSchemaRepo.FindByID(ctx, *inp.SchemaId, scenes) + if schemaId != nil { + dss, err := i.datasetSchemaRepo.FindByID(ctx, *schemaId, scenes) if err != nil { return nil, err } @@ -216,7 +244,7 @@ func (i *Dataset) ImportDataset(ctx context.Context, inp interfaces.ImportDatase if err != nil { return nil, err } - toreplace, err := i.datasetRepo.FindBySchemaAll(ctx, *inp.SchemaId) + toreplace, err := i.datasetRepo.FindBySchemaAll(ctx, *schemaId) if err != nil { return nil, err } @@ -225,7 +253,7 @@ func (i *Dataset) ImportDataset(ctx context.Context, inp interfaces.ImportDatase return nil, err } } else { - err = csv.GuessSchema(inp.SceneId) + err = csv.GuessSchema(sceneId) if err != nil { return nil, err } @@ -245,8 +273,8 @@ func (i *Dataset) ImportDataset(ctx context.Context, inp interfaces.ImportDatase return nil, err } - if inp.SchemaId != nil { - layergroups, err := i.layerRepo.FindGroupBySceneAndLinkedDatasetSchema(ctx, inp.SceneId, *inp.SchemaId) + if schemaId != nil { + layergroups, err := i.layerRepo.FindGroupBySceneAndLinkedDatasetSchema(ctx, sceneId, *schemaId) if err != nil { return nil, err } @@ -279,7 +307,7 @@ func (i *Dataset) ImportDataset(ctx context.Context, inp interfaces.ImportDatase name = rf.Value().Value().(string) } layerItem, layerProperty, err := initializer.LayerItem{ - SceneID: inp.SceneId, + SceneID: sceneId, ParentLayerID: lg.ID(), Plugin: builtin.Plugin(), ExtensionID: &extensionForLinkedLayers, diff --git a/internal/usecase/interfaces/dataset.go b/internal/usecase/interfaces/dataset.go index 8a87593d..9f6eee02 100644 --- a/internal/usecase/interfaces/dataset.go +++ b/internal/usecase/interfaces/dataset.go @@ -35,6 +35,14 @@ type ImportDatasetParam struct { SchemaId *id.DatasetSchemaID } +type ImportDatasetFromGoogleSheetParam struct { + Token string + FileID string + SheetName string + SceneId id.SceneID + SchemaId *id.DatasetSchemaID +} + type RemoveDatasetSchemaParam struct { SchemaId id.DatasetSchemaID Force *bool @@ -56,6 +64,7 @@ type Dataset interface { GraphFetch(context.Context, id.DatasetID, int, *usecase.Operator) (dataset.List, error) FetchSchema(context.Context, []id.DatasetSchemaID, *usecase.Operator) (dataset.SchemaList, error) ImportDataset(context.Context, ImportDatasetParam, *usecase.Operator) (*dataset.Schema, error) + ImportDatasetFromGoogleSheet(context.Context, ImportDatasetFromGoogleSheetParam, *usecase.Operator) (*dataset.Schema, error) GraphFetchSchema(context.Context, id.DatasetSchemaID, int, *usecase.Operator) (dataset.SchemaList, error) AddDynamicDatasetSchema(context.Context, AddDynamicDatasetSchemaParam) (*dataset.Schema, error) AddDynamicDataset(context.Context, AddDynamicDatasetParam) (*dataset.Schema, *dataset.Dataset, error) diff --git a/schema.graphql b/schema.graphql index d2ee5cd3..9d5d82fa 100644 --- a/schema.graphql +++ b/schema.graphql @@ -1054,6 +1054,14 @@ input ImportDatasetInput { datasetSchemaId: ID } +input ImportDatasetFromGoogleSheetInput { + accessToken: String! + fileId: String! + sheetName: String! + sceneId: ID! + datasetSchemaId: ID +} + input AddDatasetSchemaInput { sceneId: ID! name: String! @@ -1418,6 +1426,7 @@ type Mutation { input: RemoveDatasetSchemaInput! ): RemoveDatasetSchemaPayload importDataset(input: ImportDatasetInput!): ImportDatasetPayload + importDatasetFromGoogleSheet(input: ImportDatasetFromGoogleSheetInput!): ImportDatasetPayload addDatasetSchema(input: AddDatasetSchemaInput!): AddDatasetSchemaPayload # Property