new, edit, delete duckdb (#145)

* new, edit, delete duckdb

* duckdb new/edit works

* merged

* show code when created

* just needed to clear cookies

* no edit, copy and paste api

* delete checks team

* update ui for code

* Update router.go

* remove all cookies on logout

* improve styles

* add static files

* show api key at top

* add flashes

* add changes and merge

* database returns models

---------

Co-authored-by: poundifdef <jay@scratchdata.com>

go.mod

@@ -39,7 +39,7 @@ require (
 	google.golang.org/api v0.170.0
 	gorm.io/driver/postgres v1.5.7
 	gorm.io/driver/sqlite v1.5.5
-	gorm.io/gorm v1.25.7
+	gorm.io/gorm v1.25.9
 )
 
 require (
@@ -48,6 +48,7 @@ require (
 	cloud.google.com/go/compute/metadata v0.2.3 // indirect
 	cloud.google.com/go/iam v1.1.6 // indirect
 	dario.cat/mergo v1.0.0 // indirect
+	filippo.io/edwards25519 v1.1.0 // indirect
 	github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect
 	github.com/BurntSushi/toml v1.3.2 // indirect
 	github.com/ClickHouse/ch-go v0.61.3 // indirect
@@ -84,7 +85,7 @@ require (
 	github.com/go-faster/errors v0.7.1 // indirect
 	github.com/go-logr/logr v1.4.1 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
-	github.com/go-sql-driver/mysql v1.7.0 // indirect
+	github.com/go-sql-driver/mysql v1.8.1 // indirect
 	github.com/goccy/go-json v0.10.2 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
@@ -94,6 +95,9 @@ require (
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
 	github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
 	github.com/googleapis/gax-go/v2 v2.12.2 // indirect
+	github.com/gorilla/csrf v1.7.2 // indirect
+	github.com/gorilla/securecookie v1.1.2 // indirect
+	github.com/gorilla/sessions v1.2.2 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
 	github.com/jackc/pgx/v5 v5.4.3 // indirect
@@ -153,5 +157,7 @@ require (
 	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
+	gorm.io/datatypes v1.2.0 // indirect
+	gorm.io/driver/mysql v1.5.6 // indirect
 	olympos.io/encoding/edn v0.0.0-20201019073823-d3554ca0b0a3 // indirect
 )

go.sum

@@ -17,6 +17,8 @@ cloud.google.com/go/storage v1.37.0 h1:WI8CsaFO8Q9KjPVtsZ5Cmi0dXV25zMoX0FklT7c3J
 cloud.google.com/go/storage v1.37.0/go.mod h1:i34TiT2IhiNDmcj65PqwCjcoUX7Z5pLzS8DEmoiFq1k=
 dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk=
 dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
+filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
+filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
 github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0=
 github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
@@ -144,6 +146,8 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
 github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc=
 github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
+github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y=
+github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
 github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
 github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
@@ -194,6 +198,12 @@ github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfF
 github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0=
 github.com/googleapis/gax-go/v2 v2.12.2 h1:mhN09QQW1jEWeMF74zGR81R30z4VJzjZsfkUhuHF+DA=
 github.com/googleapis/gax-go/v2 v2.12.2/go.mod h1:61M8vcyyXR2kqKFxKrfA22jaA8JGF7Dc8App1U3H6jc=
+github.com/gorilla/csrf v1.7.2 h1:oTUjx0vyf2T+wkrx09Trsev1TE+/EbDAeHtSTbtC2eI=
+github.com/gorilla/csrf v1.7.2/go.mod h1:F1Fj3KG23WYHE6gozCmBAezKookxbIvUJT+121wTuLk=
+github.com/gorilla/securecookie v1.1.2 h1:YCIWL56dvtr73r6715mJs5ZvhtnY73hBvEF8kXD8ePA=
+github.com/gorilla/securecookie v1.1.2/go.mod h1:NfCASbcHqRSY+3a8tlWJwsQap2VX5pwzwo4h3eOamfo=
+github.com/gorilla/sessions v1.2.2 h1:lqzMYz6bOfvn2WriPUjNByzeXIlVzURcPmgMczkmTjY=
+github.com/gorilla/sessions v1.2.2/go.mod h1:ePLdVu+jbEgHH+KWw8I1z2wqd0BAdAQh/8LRvBeoNcQ=
 github.com/ilyakaznacheev/cleanenv v1.5.0 h1:0VNZXggJE2OYdXE87bfSSwGxeiGt9moSR2lOrsHHvr4=
 github.com/ilyakaznacheev/cleanenv v1.5.0/go.mod h1:a5aDzaJrLCQZsazHol1w8InnDcOX0OColm64SlIi6gk=
 github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
@@ -517,12 +527,18 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gorm.io/datatypes v1.2.0 h1:5YT+eokWdIxhJgWHdrb2zYUimyk0+TaFth+7a0ybzco=
+gorm.io/datatypes v1.2.0/go.mod h1:o1dh0ZvjIjhH/bngTpypG6lVRJ5chTBxE09FH/71k04=
+gorm.io/driver/mysql v1.5.6 h1:Ld4mkIickM+EliaQZQx3uOJDJHtrd70MxAUqWqlx3Y8=
+gorm.io/driver/mysql v1.5.6/go.mod h1:sEtPWMiqiN1N1cMXoXmBbd8C6/l+TESwriotuRRpkDM=
 gorm.io/driver/postgres v1.5.7 h1:8ptbNJTDbEmhdr62uReG5BGkdQyeasu/FZHxI0IMGnM=
 gorm.io/driver/postgres v1.5.7/go.mod h1:3e019WlBaYI5o5LIdNV+LyxCMNtLOQETBXL2h4chKpA=
 gorm.io/driver/sqlite v1.5.5 h1:7MDMtUZhV065SilG62E0MquljeArQZNfJnjd9i9gx3E=
 gorm.io/driver/sqlite v1.5.5/go.mod h1:6NgQ7sQWAIFsPrJJl1lSNSu2TABh0ZZ/zm5fosATavE=
 gorm.io/gorm v1.25.7 h1:VsD6acwRjz2zFxGO50gPO6AkNs7KKnvfzUjHQhZDz/A=
 gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8=
+gorm.io/gorm v1.25.9 h1:wct0gxZIELDk8+ZqF/MVnHLkA1rvYlBWUMv2EdsK1g8=
+gorm.io/gorm v1.25.9/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8=
 gotest.tools/v3 v3.3.0 h1:MfDY1b1/0xN1CyMlQDac0ziEy9zJQd9CXBRRDHw2jJo=
 gotest.tools/v3 v3.3.0/go.mod h1:Mcr9QNxkg0uMvy/YElmo4SpXgJKWgQvYrT7Kw5RzJ1A=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

main.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"embed"
+	"github.com/scratchdata/scratchdata/pkg/api"
 	"github.com/scratchdata/scratchdata/pkg/app"
 	"github.com/scratchdata/scratchdata/pkg/config"
 	"github.com/scratchdata/scratchdata/pkg/storage"
@@ -58,10 +59,12 @@ func main() {
 		log.Fatal().Err(err).Msg("Unable to set up data sink")
 	}
 
-	mux, err := app.GetMux(storageServices, destinationManager, dataSink, configOptions)
+	apiFunctions, err := api.NewScratchDataAPI(storageServices, destinationManager, dataSink, configOptions)
 	if err != nil {
 		log.Fatal().Err(err).Msg("Unable to build API")
 	}
 
+	mux := api.CreateMux(storageServices, apiFunctions, configOptions, destinationManager)
+
 	app.Run(configOptions, storageServices, destinationManager, dataSink, mux)
 }

pkg/api/auth.go

@@ -118,8 +118,11 @@ func (a *ScratchDataAPIStruct) Authenticator(ja *jwtauth.JWTAuth) func(http.Hand
 }
 
 func (a *ScratchDataAPIStruct) Logout(w http.ResponseWriter, r *http.Request) {
-	jwtCookie := &http.Cookie{Name: "jwt", Value: "", HttpOnly: true, Path: "/"}
-	http.SetCookie(w, jwtCookie)
+	for _, cookie := range r.Cookies() {
+		cookie.MaxAge = 0
+		cookie.Value = ""
+		http.SetCookie(w, cookie)
+	}
 	http.Redirect(w, r, "login", http.StatusSeeOther)
 }
 

pkg/api/data.go

@@ -40,9 +40,11 @@ func (a *ScratchDataAPIStruct) Copy(w http.ResponseWriter, r *http.Request) {
 
 	message.SourceID = a.AuthGetDatabaseID(r.Context())
 
+	teamId := a.AuthGetTeamID(r.Context())
+
 	// Make sure the destination db is the same team as the source
-	dest := a.storageServices.Database.GetDestination(r.Context(), message.DestinationID)
-	if dest.TeamID != a.AuthGetTeamID(r.Context()) {
+	_, err = a.storageServices.Database.GetDestination(r.Context(), teamId, message.DestinationID)
+	if err != nil {
 		http.Error(w, "invalid destination", http.StatusBadRequest)
 		return
 	}

pkg/api/destinations.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/scratchdata/scratchdata/pkg/config"
 	"github.com/scratchdata/scratchdata/pkg/storage/database/models"
+	"gorm.io/datatypes"
 
 	"github.com/go-chi/render"
 	"github.com/google/uuid"
@@ -26,10 +27,13 @@ func (a *ScratchDataAPIStruct) GetDestinations(w http.ResponseWriter, r *http.Re
 		http.Error(w, "unable to get user", http.StatusInternalServerError)
 		return
 	}
-	dest := a.storageServices.Database.GetDestinations(r.Context(), user.ID)
+	dest, err := a.storageServices.Database.GetDestinations(r.Context(), user.ID)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
 	for i := range dest {
-		dest[i].APIKeys = nil
-		dest[i].Settings = nil
+		dest[i].Settings = datatypes.NewJSONType(map[string]any{})
 	}
 	render.JSON(w, r, dest)
 }
@@ -56,7 +60,7 @@ func (a *ScratchDataAPIStruct) CreateDestination(w http.ResponseWriter, r *http.
 		http.Error(w, "unable to get user", http.StatusInternalServerError)
 		return
 	}
-	newDest, err := a.storageServices.Database.CreateDestination(r.Context(), user.ID, dest.Type, dest.Settings)
+	newDest, err := a.storageServices.Database.CreateDestination(r.Context(), user.ID, dest.Name, dest.Type, dest.Settings)
 	if err != nil {
 		render.Status(r, http.StatusInternalServerError)
 		render.PlainText(w, r, err.Error())

pkg/api/router.go

@@ -7,6 +7,8 @@ import (
 	"github.com/go-chi/cors"
 	"github.com/go-chi/jwtauth/v5"
 	"github.com/scratchdata/scratchdata/pkg/config"
+	"github.com/scratchdata/scratchdata/pkg/destinations"
+	"github.com/scratchdata/scratchdata/pkg/storage"
 	"github.com/scratchdata/scratchdata/pkg/view"
 	"github.com/scratchdata/scratchdata/pkg/view/static"
 
@@ -27,7 +29,12 @@ var responseSize = promauto.NewHistogramVec(prometheus.HistogramOpts{
 	Buckets: prometheus.ExponentialBucketsRange(1000, 100_000_000, 20),
 }, []string{"route"})
 
-func CreateMux(apiFunctions *ScratchDataAPIStruct, c config.ScratchDataConfig) *chi.Mux {
+func CreateMux(
+	storageServices *storage.Services,
+	apiFunctions *ScratchDataAPIStruct,
+	c config.ScratchDataConfig,
+	destinationManager *destinations.DestinationManager,
+) *chi.Mux {
 	r := chi.NewRouter()
 	r.Use(PrometheusMiddleware)
 	r.Get("/healthcheck", apiFunctions.Healthcheck)
@@ -71,7 +78,7 @@ func CreateMux(apiFunctions *ScratchDataAPIStruct, c config.ScratchDataConfig) *
 	})
 
 	if c.Dashboard.Enabled {
-		d, err := view.New(c.Dashboard, apiFunctions.Authenticator(apiFunctions.tokenAuth))
+		d, err := view.New(storageServices, c.Dashboard, destinationManager, apiFunctions.Authenticator(apiFunctions.tokenAuth))
 		if err != nil {
 			panic(err)
 		}

pkg/app/app.go

@@ -3,16 +3,15 @@ package app
 import (
 	"context"
 	"fmt"
+	"github.com/scratchdata/scratchdata/pkg/config"
+	"github.com/scratchdata/scratchdata/pkg/storage"
 	"net/http"
 	"os"
 	"os/signal"
 	"strconv"
 	"sync"
 	"syscall"
 
-	"github.com/scratchdata/scratchdata/pkg/config"
-	"github.com/scratchdata/scratchdata/pkg/storage"
-
 	"github.com/go-chi/chi/v5"
 	"github.com/go-chi/chi/v5/middleware"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
@@ -66,22 +65,6 @@ func setupLogs(logConfig config.Logging) {
 	}
 }
 
-func GetMux(
-	storageServices *storage.Services,
-	destinationManager *destinations.DestinationManager,
-	dataSink datasink.DataSink,
-	c config.ScratchDataConfig,
-) (*chi.Mux, error) {
-	apiFunctions, err := api.NewScratchDataAPI(storageServices, destinationManager, dataSink, c)
-	if err != nil {
-		log.Error().Err(err).Msg("Unable to start API")
-		return nil, err
-	}
-
-	mux := api.CreateMux(apiFunctions, c)
-	return mux, nil
-}
-
 func Run(
 	config config.ScratchDataConfig,
 	storageServices *storage.Services,

pkg/config/config.go

@@ -90,6 +90,8 @@ type ScratchDataConfig struct {
 type DashboardConfig struct {
 	Enabled            bool   `yaml:"enabled"`
 	LiveReload         bool   `yaml:"live_reload"`
+	CSRFSecret         string `yaml:"csrf_secret"`
+	ExternalURL        string `yaml:"external_url"`
 	GoogleRedirectURL  string `yaml:"google_redirect_url"`
 	GoogleClientID     string `yaml:"google_client_id"`
 	GoogleClientSecret string `yaml:"google_client_secret"`

pkg/destinations/destinations.go

@@ -100,15 +100,16 @@ func (m *DestinationManager) Destination(ctx context.Context, databaseID int64)
 			return nil, err
 		}
 
+		settings := creds.Settings.Data()
 		switch creds.Type {
 		case "duckdb":
-			dest, err = duckdb.OpenServer(creds.Settings)
+			dest, err = duckdb.OpenServer(settings)
 		case "clickhouse":
-			dest, err = clickhouse.OpenServer(creds.Settings)
+			dest, err = clickhouse.OpenServer(settings)
 		case "redshift":
-			dest, err = redshift.OpenServer(creds.Settings)
+			dest, err = redshift.OpenServer(settings)
 		case "bigquery":
-			dest, err = bigquery.OpenServer(creds.Settings)
+			dest, err = bigquery.OpenServer(settings)
 		}
 
 		if err != nil {

pkg/storage/database/database.go

@@ -15,10 +15,11 @@ import (
 type Database interface {
 	VerifyAdminAPIKey(ctx context.Context, hashedAPIKey string) bool
 
-	GetDestinations(ctx context.Context, teamId uint) []config.Destination
-	GetDestination(ctx context.Context, destId uint) models.Destination
-	CreateDestination(ctx context.Context, teamId uint, destType string, settings map[string]any) (config.Destination, error)
-	GetDestinationCredentials(ctx context.Context, dbID int64) (config.Destination, error)
+	GetDestinations(ctx context.Context, teamId uint) ([]models.Destination, error)
+	GetDestination(ctx context.Context, teamId, destId uint) (models.Destination, error)
+	CreateDestination(ctx context.Context, teamId uint, name string, destType string, settings map[string]any) (config.Destination, error)
+	DeleteDestination(ctx context.Context, teamId uint, destId int64) error
+	GetDestinationCredentials(ctx context.Context, dbID int64) (models.Destination, error)
 
 	AddAPIKey(ctx context.Context, destId int64, hashedAPIKey string) error
 	GetAPIKeyDetails(ctx context.Context, hashedAPIKey string) (models.APIKey, error)
@@ -27,6 +28,7 @@ type Database interface {
 	GetShareQuery(ctx context.Context, queryId uuid.UUID) (models.SharedQuery, bool)
 
 	GetUser(int64) *models.User
+	GetTeamId(userId uint) (uint, error)
 	CreateUser(email string, source string, details string) (*models.User, error)
 
 	Hash(s string) string