feat(server): export dataset as csv (#409)

reearth · May 10, 2023 · 79077b7 · 79077b7
1 parent 2a97eff
commit 79077b7
Show file tree

Hide file tree

Showing 29 changed files with 461 additions and 55 deletions.
diff --git a/server/e2e/common_test.go → server/e2e/common.go b/server/e2e/common_test.go → server/e2e/common.go
@@ -8,23 +8,51 @@ import (
 
 	"github.com/gavv/httpexpect/v2"
 	"github.com/reearth/reearth/server/internal/app"
+	"github.com/reearth/reearth/server/internal/infrastructure/fs"
 	"github.com/reearth/reearth/server/internal/infrastructure/memory"
 	"github.com/reearth/reearth/server/internal/infrastructure/mongo"
 	"github.com/reearth/reearth/server/internal/usecase/gateway"
 	"github.com/reearth/reearth/server/internal/usecase/repo"
 	"github.com/reearth/reearthx/mongox/mongotest"
 	"github.com/samber/lo"
+	"github.com/spf13/afero"
 )
 
+type Seeder func(ctx context.Context, r *repo.Container) error
+
 func init() {
 	mongotest.Env = "REEARTH_DB"
 }
 
-func StartServer(t *testing.T, cfg *app.Config, useMongo bool) *httpexpect.Expect {
+func StartServer(t *testing.T, cfg *app.Config, useMongo bool, seeder Seeder) *httpexpect.Expect {
+	e, _ := StartServerAndRepos(t, cfg, useMongo, seeder)
+	return e
+}
+
+func StartServerAndRepos(t *testing.T, cfg *app.Config, useMongo bool, seeder Seeder) (*httpexpect.Expect, *repo.Container) {
+	ctx := context.Background()
+
+	var repos *repo.Container
+	if useMongo {
+		db := mongotest.Connect(t)(t)
+		repos = lo.Must(mongo.New(ctx, db, false))
+	} else {
+		repos = memory.New()
+	}
+
+	if seeder != nil {
+		if err := seeder(ctx, repos); err != nil {
+			t.Fatalf("failed to seed the db: %s", err)
+		}
+	}
+
+	return StartServerWithRepos(t, cfg, repos), repos
+}
+func StartServerWithRepos(t *testing.T, cfg *app.Config, repos *repo.Container) *httpexpect.Expect {
 	t.Helper()
 
 	if testing.Short() {
-		t.SkipNow()
+		t.Skip("skipping test in short mode.")
 	}
 
 	ctx := context.Background()
@@ -34,18 +62,13 @@ func StartServer(t *testing.T, cfg *app.Config, useMongo bool) *httpexpect.Expec
 		t.Fatalf("server failed to listen: %v", err)
 	}
 
-	var repos *repo.Container
-	if useMongo {
-		db := mongotest.Connect(t)(t)
-		repos = lo.Must(mongo.New(ctx, db, false))
-	} else {
-		repos = memory.New()
-	}
-
 	srv := app.NewServer(ctx, &app.ServerConfig{
-		Config:   cfg,
-		Repos:    repos,
-		Gateways: &gateway.Container{},
+		Config: cfg,
+		Repos:  repos,
+		Gateways: &gateway.Container{
+			File: lo.Must(fs.NewFile(afero.NewMemMapFs(), "https://example.com")),
+		},
+		Debug: true,
 	})
 
 	ch := make(chan error)

diff --git a/server/e2e/dataset_export_test.go b/server/e2e/dataset_export_test.go
@@ -0,0 +1,44 @@
+package e2e
+
+import (
+	"net/http"
+	"testing"
+
+	"github.com/reearth/reearth/server/internal/app"
+	"github.com/reearth/reearth/server/pkg/dataset"
+)
+
+func TestDatasetExport(t *testing.T) {
+	e := StartServer(t, &app.Config{
+		Origins: []string{"https://example.com"},
+		AuthSrv: app.AuthSrvConfig{
+			Disabled: true,
+		},
+	}, true, baseSeeder)
+
+	e.GET("/api/datasets/test").
+		WithHeader("Origin", "https://example.com").
+		Expect().
+		Status(http.StatusUnauthorized)
+
+	e.GET("/api/datasets/test").
+		WithHeader("Origin", "https://example.com").
+		WithHeader("X-Reearth-Debug-User", uId.String()).
+		Expect().
+		Status(http.StatusNotFound)
+
+	e.GET("/api/datasets/"+dataset.NewID().String()).
+		WithHeader("Origin", "https://example.com").
+		WithHeader("X-Reearth-Debug-User", uId.String()).
+		Expect().
+		Status(http.StatusNotFound)
+
+	res := e.GET("/api/datasets/"+dssId.String()).
+		WithHeader("Origin", "https://example.com").
+		WithHeader("X-Reearth-Debug-User", uId.String()).
+		Expect().
+		Status(http.StatusOK).
+		ContentType("text/csv")
+	res.Header("Content-Disposition").Equal("attachment;filename=test.csv")
+	res.Body().Equal("f1,f2,f3,location\ntest,123,true,\"12.000000, 11.000000\"\n")
+}
diff --git a/server/e2e/ping_test.go b/server/e2e/ping_test.go
@@ -13,7 +13,7 @@ func TestPingAPI(t *testing.T) {
 		AuthSrv: app.AuthSrvConfig{
 			Disabled: true,
 		},
-	}, false)
+	}, false, nil)
 
 	e.OPTIONS("/api/ping").
 		WithHeader("Origin", "https://example.com").

diff --git a/server/e2e/seeder.go b/server/e2e/seeder.go
@@ -0,0 +1,93 @@
+package e2e
+
+import (
+	"context"
+	"net/url"
+	"time"
+
+	"github.com/reearth/reearth/server/internal/usecase/repo"
+	"github.com/reearth/reearth/server/pkg/dataset"
+	"github.com/reearth/reearth/server/pkg/id"
+	"github.com/reearth/reearth/server/pkg/project"
+	"github.com/reearth/reearth/server/pkg/user"
+	"github.com/reearth/reearth/server/pkg/workspace"
+	"github.com/reearth/reearthx/util"
+	"github.com/samber/lo"
+)
+
+var (
+	uId    = user.NewID()
+	wId    = workspace.NewID()
+	pId    = id.NewProjectID()
+	pAlias = "PROJECT_ALIAS"
+
+	sId   = id.NewSceneID()
+	dssId = dataset.NewSchemaID()
+
+	now = time.Date(2022, time.January, 1, 0, 0, 0, 0, time.UTC)
+)
+
+func baseSeeder(ctx context.Context, r *repo.Container) error {
+	defer util.MockNow(now)()
+
+	u := user.New().
+		ID(uId).
+		Workspace(wId).
+		Name("e2e").
+		Email("e2e@e2e.com").
+		MustBuild()
+	if err := r.User.Save(ctx, u); err != nil {
+		return err
+	}
+
+	w := workspace.New().ID(wId).
+		Name("e2e").
+		Personal(false).
+		Members(map[id.UserID]workspace.Role{u.ID(): workspace.RoleOwner}).
+		MustBuild()
+	if err := r.Workspace.Save(ctx, w); err != nil {
+		return err
+	}
+
+	p := project.New().ID(pId).
+		Name("p1").
+		Description("p1 desc").
+		ImageURL(lo.Must(url.Parse("https://test.com"))).
+		Workspace(w.ID()).
+		Alias(pAlias).
+		MustBuild()
+	if err := r.Project.Save(ctx, p); err != nil {
+		return err
+	}
+
+	fId1, fId2, fId3, fId4 := dataset.NewFieldID(), dataset.NewFieldID(), dataset.NewFieldID(), dataset.NewFieldID()
+	dss := dataset.NewSchema().ID(dssId).
+		Name("test.csv").
+		Scene(sId).
+		Dynamic(false).
+		Fields([]*dataset.SchemaField{
+			dataset.NewSchemaField().ID(fId1).Name("f1").Type(dataset.ValueTypeString).MustBuild(),
+			dataset.NewSchemaField().ID(fId2).Name("f2").Type(dataset.ValueTypeNumber).MustBuild(),
+			dataset.NewSchemaField().ID(fId3).Name("f3").Type(dataset.ValueTypeBool).MustBuild(),
+			dataset.NewSchemaField().ID(fId4).Name("location").Type(dataset.ValueTypeLatLng).MustBuild(),
+		}).
+		Source("file:///dss.csv").
+		MustBuild()
+	if err := r.DatasetSchema.Save(ctx, dss); err != nil {
+		return err
+	}
+
+	ds := dataset.New().ID(id.NewDatasetID()).Schema(dss.ID()).Scene(sId).
+		Fields([]*dataset.Field{
+			dataset.NewField(fId1, dataset.ValueTypeString.ValueFrom("test"), ""),
+			dataset.NewField(fId2, dataset.ValueTypeNumber.ValueFrom(123), ""),
+			dataset.NewField(fId3, dataset.ValueTypeBool.ValueFrom(true), ""),
+			dataset.NewField(fId4, dataset.ValueTypeLatLng.ValueFrom(dataset.LatLng{Lat: 11, Lng: 12}), ""),
+		}).
+		MustBuild()
+	if err := r.Dataset.Save(ctx, ds); err != nil {
+		return err
+	}
+
+	return nil
+}
diff --git a/server/internal/adapter/http/export.go b/server/internal/adapter/http/export.go
@@ -0,0 +1,32 @@
+package http
+
+import (
+	"net/http"
+
+	"github.com/labstack/echo/v4"
+	"github.com/reearth/reearth/server/internal/adapter"
+	"github.com/reearth/reearth/server/pkg/id"
+	"github.com/reearth/reearthx/rerror"
+)
+
+func ExportDataset() echo.HandlerFunc {
+	return func(c echo.Context) error {
+		ctx := c.Request().Context()
+		u := adapter.Usecases(ctx)
+
+		param := c.Param("datasetSchemaId")
+
+		dssId, err := id.DatasetSchemaIDFrom(param)
+		if err != nil {
+			return rerror.ErrNotFound
+		}
+
+		r, name, err := u.Dataset.Export(ctx, dssId, "csv", nil)
+		if err != nil {
+			return err
+		}
+
+		c.Response().Header().Add("Content-Disposition", "attachment;filename="+name)
+		return c.Stream(http.StatusOK, "text/csv", r)
+	}
+}
diff --git a/server/internal/app/app.go b/server/internal/app/app.go
@@ -11,6 +11,7 @@ import (
 	"github.com/99designs/gqlgen/graphql/playground"
 	"github.com/labstack/echo/v4"
 	"github.com/labstack/echo/v4/middleware"
+	http2 "github.com/reearth/reearth/server/internal/adapter/http"
 	"github.com/reearth/reearth/server/internal/usecase/interactor"
 	"github.com/reearth/reearthx/log"
 	"github.com/reearth/reearthx/rerror"
@@ -106,6 +107,7 @@ func initEcho(ctx context.Context, cfg *ServerConfig) *echo.Echo {
 	apiPrivate := api.Group("", private)
 	apiPrivate.POST("/graphql", GraphqlAPI(cfg.Config.GraphQL, gqldev))
 	apiPrivate.GET("/layers/:param", ExportLayer(), AuthRequiredMiddleware())
+	apiPrivate.GET("/datasets/:datasetSchemaId", http2.ExportDataset(), AuthRequiredMiddleware())
 	apiPrivate.POST("/signup", Signup())
 
 	if !cfg.Config.AuthSrv.Disabled {