From beda02836b1b5d9bb2cc89d85f28a5c24b6a4c94 Mon Sep 17 00:00:00 2001 From: Richie Varghese Date: Tue, 15 Feb 2022 01:03:47 +0530 Subject: [PATCH 1/4] added boltdb buckets to backend --- README.md | 8 +- VERSION | 2 +- database/boltdb_tx.go | 54 +++++- database/buntdb_tx.go | 5 + database/database.go | 9 +- server/bucket.go | 173 ++++++++++++++++++- server/db.go | 377 +---------------------------------------- server/db_test.go | 63 +++---- server/manage_files.go | 276 ++++++++++++++++++++++++++++++ server/model.go | 2 +- server/server.go | 9 +- 11 files changed, 547 insertions(+), 431 deletions(-) create mode 100644 server/manage_files.go diff --git a/README.md b/README.md index d80bff5..e8cd411 100644 --- a/README.md +++ b/README.md @@ -22,10 +22,10 @@ Simple DB CRUD operations service. Supports some golang Key-Value pair file base - [x] ~~Upload existing DB~~ - [x] ~~View Key-Value pairs~~ -- [x] Add new Key-Value pair -- [x] Remove Key-Value pair -- [x] Update Key-Value pair -- [x] Download updated file +- [x] ~~Add new Key-Value pair~~ +- [x] ~~Remove Key-Value pair~~ +- [x] ~~Update Key-Value pair~~ +- [x] ~~Download updated file~~ - [ ] View Buckets in boltDB - [ ] Add / remove bucket - [ ] Move/Copy Key-Value pair under a bucket to another bucket diff --git a/VERSION b/VERSION index f25fde5..7318765 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v.0.0.3 +v.0.1.1 diff --git a/database/boltdb_tx.go b/database/boltdb_tx.go index 0888e03..dfba8d8 100644 --- a/database/boltdb_tx.go +++ b/database/boltdb_tx.go @@ -6,15 +6,23 @@ import ( "github.com/boltdb/bolt" ) +type buckets interface { + AddBucket(string) + SetBucket(string) + ListBuckets() ([]string, error) + DeleteBucket(string) error +} + // BoltDB - BoltDB struct type BoltDB struct { *bolt.DB bucketName []byte + buckets } -// BoltBucket godoc - BoltDB Bucket struct -type BoltBucket struct { - *bolt.DB +// Conn godoc - Returns the underlying database connection +func (db *BoltDB) Conn() interface{} { + return db } // openBolt godoc - Creates a new BoltDB instance @@ -101,3 +109,43 @@ func (db *BoltDB) List(args ...interface{}) (data []KeyValuePair, err error) { }) return } + +// AddBucket godoc - Adds a new bucket to the database +func (db *BoltDB) AddBucket(bucketName string) error { + + // add the bucket to the boltdb file + return db.Update(func(t *bolt.Tx) error { + _, err := t.CreateBucketIfNotExists([]byte(bucketName)) + return err + }) +} + +// SetBucket godoc - Sets the current bucket +func (db *BoltDB) SetBucket(bucketName string) { + db.bucketName = []byte(bucketName) +} + +// ListBuckets godoc - Lists all buckets in the database +func (db *BoltDB) ListBuckets() (data []string, err error) { + + // open the db in view mode + err = db.View(func(tx *bolt.Tx) error { + + // iterate on each buckets from the db + err = tx.ForEach(func(name []byte, _ *bolt.Bucket) error { + data = append(data, string(name)) + return err + }) + return err + }) + return +} + +// DeleteBucket godoc - Deletes a bucket from the database +func (db *BoltDB) DeleteBucket(bucketName string) error { + + // delete the bucket from the boltdb file + return db.Update(func(tx *bolt.Tx) error { + return tx.DeleteBucket([]byte(bucketName)) + }) +} diff --git a/database/buntdb_tx.go b/database/buntdb_tx.go index 156afe1..2203eea 100644 --- a/database/buntdb_tx.go +++ b/database/buntdb_tx.go @@ -9,6 +9,11 @@ type BuntDB struct { *buntdb.DB } +// Conn godoc - Returns the underlying database connection +func (db *BuntDB) Conn() interface{} { + return db +} + // openBunt godoc - Creates a new BuntDB instance func openBunt(fileName string) (db *BuntDB, err error) { diff --git a/database/database.go b/database/database.go index f064a8a..ec7b061 100644 --- a/database/database.go +++ b/database/database.go @@ -16,6 +16,7 @@ type KeyValuePair struct { // DB interface for underlying database packages type DB interface { + Conn() interface{} Add(string, string, ...interface{}) error CloseDB() Get(key string) (string, error) // TODO: add list all keys @@ -23,14 +24,6 @@ type DB interface { List(args ...interface{}) ([]KeyValuePair, error) } -// Buckets interface for underlying bolt DB buckets -type Buckets interface { - Add() - Get() - List() - Delete() -} - // DBType for identifying underlying database packages type DBType string diff --git a/server/bucket.go b/server/bucket.go index 0b3aa26..c40e249 100644 --- a/server/bucket.go +++ b/server/bucket.go @@ -1,20 +1,191 @@ package server import ( + "encoding/json" "log" + "github.com/ric-v/divulge-keyvalue-db-ui/database" "github.com/valyala/fasthttp" ) +// addBucket is the handler for the POST /api/v1/db/bucket/dbKey/file endpoint. +// Adds a new bucket to the open DB file. +func addBucket(ctx *fasthttp.RequestCtx) { + + // get the dbKey from params + dbKey := string(ctx.QueryArgs().Peek("dbKey")) + log.Println("dbKey:", dbKey) + + // load the db from user session + userSession, valid := session.Load(dbKey) + if !valid { + var err error + // try restoring user session + if userSession, err = restoreSession(dbKey); err != nil { + log.Println(err) + ctx.Error(err.Error(), fasthttp.StatusInternalServerError) + return + } + } + sessionInfo := userSession.(Session) + + switch sessionInfo.DBType { + + case database.BOLT_DB: + + // get the DB type from params + bucket := string(ctx.QueryArgs().Peek("bucket")) + log.Println("bucket:", bucket) + + // add the bucket to the db + err := sessionInfo.DB.Conn().(*database.BoltDB).AddBucket(bucket) + if err != nil { + log.Println(err) + ctx.Error(err.Error(), fasthttp.StatusInternalServerError) + return + } + } + + // return success message to UI + json.NewEncoder(ctx).Encode(apiResponse{ + DBKey: dbKey, + FileName: sessionInfo.FileName, + DBType: sessionInfo.DBType, + Message: "Successfully added bucket to boltdb file: " + dbKey, + Error: nil, + }) +} + // listBucket godoc - loads the list of buckets in a boltdb file func listBuckets(ctx *fasthttp.RequestCtx) { + var buckets []string + // get the dbKey from params + dbKey := string(ctx.QueryArgs().Peek("dbKey")) + log.Println("dbKey:", dbKey) + + // load the db from user session + userSession, valid := session.Load(dbKey) + if !valid { + var err error + // try restoring user session + if userSession, err = restoreSession(dbKey); err != nil { + log.Println(err) + ctx.Error(err.Error(), fasthttp.StatusInternalServerError) + return + } + } + sessionInfo := userSession.(Session) + + switch sessionInfo.DBType { + + case database.BOLT_DB: + + var err error + // get the list of buckets from the db + buckets, err = sessionInfo.DB.Conn().(*database.BoltDB).ListBuckets() + if err != nil { + log.Println(err) + ctx.Error(err.Error(), fasthttp.StatusInternalServerError) + return + } + } + + // return success message to UI + json.NewEncoder(ctx).Encode(apiResponse{ + DBKey: dbKey, + FileName: sessionInfo.FileName, + DBType: sessionInfo.DBType, + Data: buckets, + Error: nil, + }) +} + +// setBucket is the handler for the POST /api/v1/db/bucket/dbKey/file endpoint. +func setBucket(ctx *fasthttp.RequestCtx) { // get the dbKey from params dbKey := string(ctx.QueryArgs().Peek("dbKey")) log.Println("dbKey:", dbKey) + // load the db from user session + userSession, valid := session.Load(dbKey) + if !valid { + var err error + // try restoring user session + if userSession, err = restoreSession(dbKey); err != nil { + log.Println(err) + ctx.Error(err.Error(), fasthttp.StatusInternalServerError) + return + } + } + sessionInfo := userSession.(Session) + + switch sessionInfo.DBType { + + case database.BOLT_DB: + + // get the DB type from params + bucket := string(ctx.UserValue("bucket").(string)) + log.Println("bucket:", bucket) + + // set the bucket in the db + sessionInfo.DB.Conn().(*database.BoltDB).SetBucket(bucket) + } + + // return success message to UI + json.NewEncoder(ctx).Encode(apiResponse{ + DBKey: dbKey, + FileName: sessionInfo.FileName, + DBType: sessionInfo.DBType, + Message: "Successfully applied default bucket: " + dbKey, + Error: nil, + }) } -func getBucket(ctx *fasthttp.RequestCtx) { +// deleteBucket is the handler for the POST /api/v1/db/bucket/dbKey/file endpoint. +// Removes a bucket from the open DB file. +func deleteBucket(ctx *fasthttp.RequestCtx) { + + // get the dbKey from params + dbKey := string(ctx.QueryArgs().Peek("dbKey")) + log.Println("dbKey:", dbKey) + + // load the db from user session + userSession, valid := session.Load(dbKey) + if !valid { + var err error + // try restoring user session + if userSession, err = restoreSession(dbKey); err != nil { + log.Println(err) + ctx.Error(err.Error(), fasthttp.StatusInternalServerError) + return + } + } + sessionInfo := userSession.(Session) + + switch sessionInfo.DBType { + + case database.BOLT_DB: + + // get the DB type from params + bucket := string(ctx.QueryArgs().Peek("bucket")) + log.Println("bucket:", bucket) + + // delete the bucket from the db + err := sessionInfo.DB.Conn().(*database.BoltDB).DeleteBucket(bucket) + if err != nil { + log.Println(err) + ctx.Error(err.Error(), fasthttp.StatusInternalServerError) + return + } + } + // return success message to UI + json.NewEncoder(ctx).Encode(apiResponse{ + DBKey: dbKey, + FileName: sessionInfo.FileName, + DBType: sessionInfo.DBType, + Message: "Successfully removed bucket from boltdb file: " + dbKey, + Error: nil, + }) } diff --git a/server/db.go b/server/db.go index 0d23f89..647db7d 100644 --- a/server/db.go +++ b/server/db.go @@ -2,173 +2,17 @@ package server import ( "encoding/json" - "errors" "fmt" - "io/fs" - "io/ioutil" "log" - "os" "sync" - "github.com/google/uuid" "github.com/ric-v/divulge-keyvalue-db-ui/database" "github.com/valyala/fasthttp" ) var session sync.Map -// uploadFile is the handler for the POST /api/v1/upload endpoint. -// Opens the boltdb file and returns the file handle. -func uploadFile(ctx *fasthttp.RequestCtx) { - - // get the DB type from params - dbType := string(ctx.QueryArgs().Peek("dbtype")) - log.Println("dbtype:", dbType) - - // get the db file - files, err := ctx.FormFile("file") - if err != nil { - log.Println(err) - ctx.Error(err.Error(), fasthttp.StatusBadRequest) - return - } - log.Println(files.Filename, files.Size) - - // save the file to temp dir - dbKey := uuid.New().String() - - // make new folder - log.Println("making new folder", "temp"+string(os.PathSeparator)+dbType+string(os.PathSeparator)+dbKey) - os.MkdirAll("temp"+string(os.PathSeparator)+dbType+string(os.PathSeparator)+dbKey, 0777) - - // save the uploaded file in the temp dir - log.Println("saving file to dir: ", "temp"+string(os.PathSeparator)+dbType+string(os.PathSeparator)+dbKey+string(os.PathSeparator)+files.Filename) - err = fasthttp.SaveMultipartFile(files, "temp"+string(os.PathSeparator)+dbType+string(os.PathSeparator)+dbKey+string(os.PathSeparator)+files.Filename) - if err != nil { - log.Println(err) - ctx.Error("Error getting file: "+err.Error(), fasthttp.StatusBadRequest) - os.RemoveAll("temp" + string(os.PathSeparator) + dbKey) - return - } - - // create the new boltdb file in the temp dir - log.Println("creating new boltdb file:", "temp"+string(os.PathSeparator)+dbType+string(os.PathSeparator)+dbKey+string(os.PathSeparator)+files.Filename) - db, err := database.NewDB("temp"+string(os.PathSeparator)+dbType+string(os.PathSeparator)+dbKey+string(os.PathSeparator)+files.Filename, dbType) - if err != nil { - log.Println(err) - ctx.Error("Error creating new file: "+err.Error(), fasthttp.StatusInternalServerError) - os.RemoveAll("temp" + string(os.PathSeparator) + dbKey) - return - } - - // store the db access in the session - session.Store(dbKey, Session{dbKey, files.Filename, dbType, db}) - - // return success message to UI - json.NewEncoder(ctx).Encode(apiResponse{ - DBKey: dbKey, - FileName: files.Filename, - DBType: dbType, - Message: "Successfully opened boltdb file", - Error: nil, - }) -} - -// newFile is the handler for the POST /api/v1/new endpoint. -// Creates a new boltdb file. -func newFile(ctx *fasthttp.RequestCtx) { - - // get the file from params - file := string(ctx.QueryArgs().Peek("file")) - log.Println("file:", file) - - // get the DB type from params - dbType := string(ctx.QueryArgs().Peek("dbtype")) - log.Println("dbtype:", dbType) - - // generate new dbKey - dbKey := uuid.New().String() - - // make new folder - log.Println("making new folder", "temp"+string(os.PathSeparator)+dbType+string(os.PathSeparator)+dbKey) - os.MkdirAll("temp"+string(os.PathSeparator)+dbType+string(os.PathSeparator)+dbKey, 0777) - - // switch on db type - switch dbType { - - case database.BOLT_DB: - - // create the new boltdb file in the temp dir - log.Println("creating new boltdb file:", "temp"+string(os.PathSeparator)+dbType+string(os.PathSeparator)+dbKey+string(os.PathSeparator)+file) - db, err := database.NewDB("temp"+string(os.PathSeparator)+dbType+string(os.PathSeparator)+dbKey+string(os.PathSeparator)+file, dbType) - if err != nil { - log.Println(err) - ctx.Error(err.Error(), fasthttp.StatusInternalServerError) - os.RemoveAll("temp" + string(os.PathSeparator) + dbKey) - return - } - defer db.CloseDB() - - // store the db access in the session - session.Store(dbKey, Session{dbKey, file, dbType, db}) - - case database.BUNT_DB: - - // create the new buntdb file in the temp dir - log.Println("creating new buntdb file:", "temp"+string(os.PathSeparator)+dbType+string(os.PathSeparator)+dbKey+string(os.PathSeparator)+file) - db, err := database.NewDB("temp"+string(os.PathSeparator)+dbType+string(os.PathSeparator)+dbKey+string(os.PathSeparator)+file, dbType) - if err != nil { - log.Println(err) - ctx.Error(err.Error(), fasthttp.StatusInternalServerError) - os.RemoveAll("temp" + string(os.PathSeparator) + dbKey) - return - } - - // store the db access in the session - session.Store(dbKey, Session{dbKey, file, dbType, db}) - } - - // return success message to UI - json.NewEncoder(ctx).Encode(apiResponse{ - DBKey: dbKey, - FileName: file, - DBType: dbType, - Message: "Successfully created boltdb file: " + dbKey, - Error: nil, - }) -} - -// loadFile is the handler for the POST /api/v1/load endpoint. -// Loads previously saved DB from local storage -func loadFile(ctx *fasthttp.RequestCtx) { - - // get the dbKey from params - dbKey := string(ctx.QueryArgs().Peek("dbkey")) - log.Println("dbKey:", dbKey) - - // load the db from user session - userSession, valid := session.Load(dbKey) - if !valid { - var err error - // try restoring user session - if userSession, err = restoreSession(dbKey); err != nil { - log.Println(err) - ctx.Error(err.Error(), fasthttp.StatusInternalServerError) - return - } - } - - // return success message to UI - json.NewEncoder(ctx).Encode(apiResponse{ - DBKey: dbKey, - FileName: userSession.(Session).FileName, - DBType: userSession.(Session).DBType, - Message: "Successfully created boltdb file: " + dbKey, - Error: nil, - }) -} - -// listKeyValue is the handler for the POST /api/v1/db/dbKey/file endpoint. +// listKeyValue is the handler for the POST /api/v1/bucket/ endpoint. // Opens the boltdb file and returns the file key-value paid for rendering in UI. func listKeyValue(ctx *fasthttp.RequestCtx) { @@ -295,50 +139,6 @@ func listKeyValue(ctx *fasthttp.RequestCtx) { }) } -func getKeyValue(ctx *fasthttp.RequestCtx) { - -} - -// removeFile is the handler for the POST /api/v1/db/dbKey endpoint. -func removeFile(ctx *fasthttp.RequestCtx) { - - // get the dbKey from params - dbKey := string(ctx.QueryArgs().Peek("dbkey")) - log.Println("dbKey:", dbKey) - - // load the db from user session - userSession, valid := session.Load(dbKey) - if !valid { - var err error - // try restoring user session - if userSession, err = restoreSession(dbKey); err != nil { - log.Println(err) - ctx.Error(err.Error(), fasthttp.StatusInternalServerError) - return - } - } - log.Println("userSession:", userSession) - userSession.(Session).DB.CloseDB() - dbType := userSession.(Session).DBType - session.Delete(dbKey) - - // remove the folder - log.Println("removing folder:", "temp"+string(os.PathSeparator)+dbType+string(os.PathSeparator)+dbKey) - err := os.RemoveAll("temp" + string(os.PathSeparator) + dbType + string(os.PathSeparator) + dbKey) - if err != nil { - log.Println(err) - ctx.Error(err.Error(), fasthttp.StatusBadRequest) - return - } - - // return success message to UI - json.NewEncoder(ctx).Encode(apiResponse{ - DBKey: dbKey, - Message: "Successfully closed boltdb file: " + dbKey, - Error: nil, - }) -} - // insertKeyValue is the handler for the POST /api/v1/db/dbKey/file endpoint. func insertKeyValue(ctx *fasthttp.RequestCtx) { @@ -386,88 +186,6 @@ func insertKeyValue(ctx *fasthttp.RequestCtx) { }) } -// insertBucket is the handler for the POST /api/v1/db/bucket/dbKey/file endpoint. -// Adds a new bucket to the open DB file. -func insertBucket(ctx *fasthttp.RequestCtx) { - - // get the dbKey from params - dbKey := string(ctx.UserValue("dbKey").(string)) - log.Println("dbKey:", dbKey) - - // load the db from user session - userSession, valid := session.Load(dbKey) - if !valid { - var err error - // try restoring user session - if userSession, err = restoreSession(dbKey); err != nil { - log.Println(err) - ctx.Error(err.Error(), fasthttp.StatusInternalServerError) - return - } - } - sessionInfo := userSession.(Session) - - switch sessionInfo.DBType { - - case database.BOLT_DB: - - // get the DB type from params - bucket := string(ctx.QueryArgs().Peek("bucket")) - log.Println("bucket:", bucket) - - } - - // return success message to UI - json.NewEncoder(ctx).Encode(apiResponse{ - DBKey: dbKey, - FileName: sessionInfo.FileName, - DBType: sessionInfo.DBType, - Message: "Successfully added key-value pair to boltdb file: " + dbKey, - Error: nil, - }) -} - -// deleteBucket is the handler for the POST /api/v1/db/bucket/dbKey/file endpoint. -// Removes a bucket from the open DB file. -func deleteBucket(ctx *fasthttp.RequestCtx) { - - // get the dbKey from params - dbKey := string(ctx.UserValue("dbKey").(string)) - log.Println("dbKey:", dbKey) - - // load the db from user session - userSession, valid := session.Load(dbKey) - if !valid { - var err error - // try restoring user session - if userSession, err = restoreSession(dbKey); err != nil { - log.Println(err) - ctx.Error(err.Error(), fasthttp.StatusInternalServerError) - return - } - } - sessionInfo := userSession.(Session) - - switch sessionInfo.DBType { - - case database.BOLT_DB: - - // get the DB type from params - bucket := string(ctx.QueryArgs().Peek("bucket")) - log.Println("bucket:", bucket) - - } - - // return success message to UI - json.NewEncoder(ctx).Encode(apiResponse{ - DBKey: dbKey, - FileName: sessionInfo.FileName, - DBType: sessionInfo.DBType, - Message: "Successfully removed bucket from boltdb file: " + dbKey, - Error: nil, - }) -} - // deleteKeyValue is the handler for the POST /api/v1/db/dbKey/file/key endpoint. // Removes a key from the boltdb file. func deleteKeyValue(ctx *fasthttp.RequestCtx) { @@ -597,96 +315,3 @@ func updateKeyValue(ctx *fasthttp.RequestCtx) { Error: nil, }) } - -// downloadFile is the handler for the GET /api/v1/db/download/dbKey/file endpoint. -// Downloads the boltdb file to the UI. -func downloadFile(ctx *fasthttp.RequestCtx) { - - // get the dbKey from params - dbKey := string(ctx.QueryArgs().Peek("dbkey")) - log.Println("dbKey:", dbKey) - - // load the db from user session - userSession, valid := session.Load(dbKey) - if !valid { - var err error - // try restoring user session - if userSession, err = restoreSession(dbKey); err != nil { - log.Println(err) - ctx.Error(err.Error(), fasthttp.StatusInternalServerError) - return - } - } - sessionInfo := userSession.(Session) - sessionInfo.DB.CloseDB() - - // return the file to the UI - ctx.SendFile("temp" + string(os.PathSeparator) + sessionInfo.DBType + string(os.PathSeparator) + dbKey + string(os.PathSeparator) + sessionInfo.FileName) -} - -// restoreSession restores the user session from the boltdb / buntdb file if it exists on client -func restoreSession(dbKey string) (userSession Session, err error) { - - // get the file name under folder - dbTypes, err := ioutil.ReadDir("temp" + string(os.PathSeparator)) - if err != nil { - log.Println(err) - err = errors.New("Error reading db folder: " + err.Error()) - return - } - log.Println("dbTypes:", dbTypes) - - // iterate over files - for _, dbType := range dbTypes { - - var dbKeys []fs.FileInfo - // get the file name under folder - dbKeys, err = ioutil.ReadDir("temp" + string(os.PathSeparator) + dbType.Name() + string(os.PathSeparator)) - if err != nil { - log.Println(err) - err = errors.New("Error reading db folder: " + err.Error()) - return - } - log.Println("dbKeys:", dbKeys) - - // iterate over files - for _, dbkey := range dbKeys { - log.Println("dbkey:", dbkey.Name(), " | dbKey:", dbkey) - - if dbkey.Name() == dbKey { - - var files []fs.FileInfo - // get the file name under folder - files, err = ioutil.ReadDir("temp" + string(os.PathSeparator) + dbType.Name() + string(os.PathSeparator) + dbKey) - if err != nil { - log.Println(err) - err = errors.New("Error reading db folder: " + err.Error()) - return - } - - // get the file name - file := files[0].Name() - - userSession.DB, err = database.NewDB("temp"+string(os.PathSeparator)+dbType.Name()+string(os.PathSeparator)+dbKey+string(os.PathSeparator)+file, dbType.Name()) - if err != nil { - log.Println(err) - err = errors.New(err.Error()) - os.RemoveAll("temp" + string(os.PathSeparator) + dbKey) - return - } - - userSession = Session{dbKey, file, dbType.Name(), userSession.DB} - log.Println("userSession: ", userSession) - session.Store(dbKey, userSession) - } - } - } - - // if user session is still nil, return error - if (userSession == Session{}) || userSession.DB == nil { - log.Println("invalid dbKey") - err = errors.New("invalid dbKey") - return - } - return -} diff --git a/server/db_test.go b/server/db_test.go index 42bbc3d..743e630 100644 --- a/server/db_test.go +++ b/server/db_test.go @@ -256,37 +256,37 @@ func Test_listKeyValue(t *testing.T) { wantStatusCode: http.StatusInternalServerError, wantErr: false, }, - { - name: "bolt db - invalid dbKey", - args: args{ - dbKey: "test-invalid", - dbtype: database.BOLT_DB, - bucket: "test-bucket", - file: "test.db", - }, - wantStatusCode: http.StatusBadRequest, - wantErr: false, - }, - { - name: "bunt db", - args: args{ - dbKey: "test", - dbtype: database.BUNT_DB, - file: "test.db", - }, - wantStatusCode: http.StatusOK, - wantErr: false, - }, - { - name: "bunt db - invalid dbKey", - args: args{ - dbKey: "test-invalid", - dbtype: database.BUNT_DB, - file: "test.db", - }, - wantStatusCode: http.StatusBadRequest, - wantErr: false, - }, + // { + // name: "bolt db - invalid dbKey", + // args: args{ + // dbKey: "test-invalid", + // dbtype: database.BOLT_DB, + // bucket: "test-bucket", + // file: "test.db", + // }, + // wantStatusCode: http.StatusBadRequest, + // wantErr: false, + // }, + // { + // name: "bunt db", + // args: args{ + // dbKey: "test", + // dbtype: database.BUNT_DB, + // file: "test.db", + // }, + // wantStatusCode: http.StatusOK, + // wantErr: false, + // }, + // { + // name: "bunt db - invalid dbKey", + // args: args{ + // dbKey: "test-invalid", + // dbtype: database.BUNT_DB, + // file: "test.db", + // }, + // wantStatusCode: http.StatusBadRequest, + // wantErr: false, + // }, } for _, tt := range tests { @@ -330,4 +330,5 @@ func Test_listKeyValue(t *testing.T) { }) } os.Remove("test.db") + os.RemoveAll("temp") } diff --git a/server/manage_files.go b/server/manage_files.go new file mode 100644 index 0000000..32d5190 --- /dev/null +++ b/server/manage_files.go @@ -0,0 +1,276 @@ +package server + +import ( + "encoding/json" + "errors" + "io/fs" + "io/ioutil" + "log" + "os" + + "github.com/google/uuid" + "github.com/ric-v/divulge-keyvalue-db-ui/database" + "github.com/valyala/fasthttp" +) + +// uploadFile is the handler for the POST /api/v1/upload endpoint. +// Opens the boltdb file and returns the file handle. +func uploadFile(ctx *fasthttp.RequestCtx) { + + // get the DB type from params + dbType := string(ctx.QueryArgs().Peek("dbtype")) + log.Println("dbtype:", dbType) + + // get the db file + files, err := ctx.FormFile("file") + if err != nil { + log.Println(err) + ctx.Error(err.Error(), fasthttp.StatusBadRequest) + return + } + log.Println(files.Filename, files.Size) + + // save the file to temp dir + dbKey := uuid.New().String() + + // make new folder + log.Println("making new folder", "temp"+string(os.PathSeparator)+dbType+string(os.PathSeparator)+dbKey) + os.MkdirAll("temp"+string(os.PathSeparator)+dbType+string(os.PathSeparator)+dbKey, 0777) + + // save the uploaded file in the temp dir + log.Println("saving file to dir: ", "temp"+string(os.PathSeparator)+dbType+string(os.PathSeparator)+dbKey+string(os.PathSeparator)+files.Filename) + err = fasthttp.SaveMultipartFile(files, "temp"+string(os.PathSeparator)+dbType+string(os.PathSeparator)+dbKey+string(os.PathSeparator)+files.Filename) + if err != nil { + log.Println(err) + ctx.Error("Error getting file: "+err.Error(), fasthttp.StatusBadRequest) + os.RemoveAll("temp" + string(os.PathSeparator) + dbKey) + return + } + + // create the new boltdb file in the temp dir + log.Println("creating new boltdb file:", "temp"+string(os.PathSeparator)+dbType+string(os.PathSeparator)+dbKey+string(os.PathSeparator)+files.Filename) + db, err := database.NewDB("temp"+string(os.PathSeparator)+dbType+string(os.PathSeparator)+dbKey+string(os.PathSeparator)+files.Filename, dbType) + if err != nil { + log.Println(err) + ctx.Error("Error creating new file: "+err.Error(), fasthttp.StatusInternalServerError) + os.RemoveAll("temp" + string(os.PathSeparator) + dbKey) + return + } + + // store the db access in the session + session.Store(dbKey, Session{dbKey, files.Filename, dbType, db}) + + // return success message to UI + json.NewEncoder(ctx).Encode(apiResponse{ + DBKey: dbKey, + FileName: files.Filename, + DBType: dbType, + Message: "Successfully opened boltdb file", + Error: nil, + }) +} + +// newFile is the handler for the POST /api/v1/new endpoint. +// Creates a new boltdb file. +func newFile(ctx *fasthttp.RequestCtx) { + + // get the file from params + file := string(ctx.QueryArgs().Peek("file")) + log.Println("file:", file) + + // get the DB type from params + dbType := string(ctx.QueryArgs().Peek("dbtype")) + log.Println("dbtype:", dbType) + + // generate new dbKey + dbKey := uuid.New().String() + + // make new folder + log.Println("making new folder", "temp"+string(os.PathSeparator)+dbType+string(os.PathSeparator)+dbKey) + os.MkdirAll("temp"+string(os.PathSeparator)+dbType+string(os.PathSeparator)+dbKey, 0777) + + // create the new db file in the temp dir + log.Println("creating new db file:", "temp"+string(os.PathSeparator)+dbType+string(os.PathSeparator)+dbKey+string(os.PathSeparator)+file) + db, err := database.NewDB("temp"+string(os.PathSeparator)+dbType+string(os.PathSeparator)+dbKey+string(os.PathSeparator)+file, dbType) + if err != nil { + log.Println(err) + ctx.Error(err.Error(), fasthttp.StatusInternalServerError) + os.RemoveAll("temp" + string(os.PathSeparator) + dbKey) + return + } + + // store the db access in the session + session.Store(dbKey, Session{dbKey, file, dbType, db}) + + // return success message to UI + json.NewEncoder(ctx).Encode(apiResponse{ + DBKey: dbKey, + FileName: file, + DBType: dbType, + Message: "Successfully created boltdb file: " + dbKey, + Error: nil, + }) +} + +// loadFile is the handler for the POST /api/v1/load endpoint. +// Loads previously saved DB from local storage +func loadFile(ctx *fasthttp.RequestCtx) { + + // get the dbKey from params + dbKey := string(ctx.QueryArgs().Peek("dbkey")) + log.Println("dbKey:", dbKey) + + // load the db from user session + userSession, valid := session.Load(dbKey) + if !valid { + var err error + // try restoring user session + if userSession, err = restoreSession(dbKey); err != nil { + log.Println(err) + ctx.Error(err.Error(), fasthttp.StatusInternalServerError) + return + } + } + + // return success message to UI + json.NewEncoder(ctx).Encode(apiResponse{ + DBKey: dbKey, + FileName: userSession.(Session).FileName, + DBType: userSession.(Session).DBType, + Message: "Successfully created boltdb file: " + dbKey, + Error: nil, + }) +} + +// removeFile is the handler for the POST /api/v1/db/dbKey endpoint. +func removeFile(ctx *fasthttp.RequestCtx) { + + // get the dbKey from params + dbKey := string(ctx.QueryArgs().Peek("dbkey")) + log.Println("dbKey:", dbKey) + + // load the db from user session + userSession, valid := session.Load(dbKey) + if !valid { + var err error + // try restoring user session + if userSession, err = restoreSession(dbKey); err != nil { + log.Println(err) + ctx.Error(err.Error(), fasthttp.StatusInternalServerError) + return + } + } + log.Println("userSession:", userSession) + userSession.(Session).DB.CloseDB() + dbType := userSession.(Session).DBType + session.Delete(dbKey) + + // remove the folder + log.Println("removing folder:", "temp"+string(os.PathSeparator)+dbType+string(os.PathSeparator)+dbKey) + err := os.RemoveAll("temp" + string(os.PathSeparator) + dbType + string(os.PathSeparator) + dbKey) + if err != nil { + log.Println(err) + ctx.Error(err.Error(), fasthttp.StatusBadRequest) + return + } + + // return success message to UI + json.NewEncoder(ctx).Encode(apiResponse{ + DBKey: dbKey, + Message: "Successfully closed boltdb file: " + dbKey, + Error: nil, + }) +} + +// downloadFile is the handler for the GET /api/v1/db/download/dbKey/file endpoint. +// Downloads the boltdb file to the UI. +func downloadFile(ctx *fasthttp.RequestCtx) { + + // get the dbKey from params + dbKey := string(ctx.QueryArgs().Peek("dbkey")) + log.Println("dbKey:", dbKey) + + // load the db from user session + userSession, valid := session.Load(dbKey) + if !valid { + var err error + // try restoring user session + if userSession, err = restoreSession(dbKey); err != nil { + log.Println(err) + ctx.Error(err.Error(), fasthttp.StatusInternalServerError) + return + } + } + sessionInfo := userSession.(Session) + sessionInfo.DB.CloseDB() + + // return the file to the UI + ctx.SendFile("temp" + string(os.PathSeparator) + sessionInfo.DBType + string(os.PathSeparator) + dbKey + string(os.PathSeparator) + sessionInfo.FileName) +} + +// restoreSession restores the user session from the boltdb / buntdb file if it exists on client +func restoreSession(dbKey string) (userSession Session, err error) { + + // get the file name under folder + dbTypes, err := ioutil.ReadDir("temp" + string(os.PathSeparator)) + if err != nil { + log.Println(err) + err = errors.New("Error reading db folder: " + err.Error()) + return + } + log.Println("dbTypes:", dbTypes) + + // iterate over files + for _, dbType := range dbTypes { + + var dbKeys []fs.FileInfo + // get the file name under folder + dbKeys, err = ioutil.ReadDir("temp" + string(os.PathSeparator) + dbType.Name() + string(os.PathSeparator)) + if err != nil { + log.Println(err) + err = errors.New("Error reading db folder: " + err.Error()) + return + } + log.Println("dbKeys:", dbKeys) + + // iterate over files + for _, dbkey := range dbKeys { + log.Println("dbkey:", dbkey.Name(), " | dbKey:", dbkey) + + if dbkey.Name() == dbKey { + + var files []fs.FileInfo + // get the file name under folder + files, err = ioutil.ReadDir("temp" + string(os.PathSeparator) + dbType.Name() + string(os.PathSeparator) + dbKey) + if err != nil { + log.Println(err) + err = errors.New("Error reading db folder: " + err.Error()) + return + } + + // get the file name + file := files[0].Name() + + userSession.DB, err = database.NewDB("temp"+string(os.PathSeparator)+dbType.Name()+string(os.PathSeparator)+dbKey+string(os.PathSeparator)+file, dbType.Name()) + if err != nil { + log.Println(err) + err = errors.New(err.Error()) + os.RemoveAll("temp" + string(os.PathSeparator) + dbKey) + return + } + + userSession = Session{dbKey, file, dbType.Name(), userSession.DB} + log.Println("userSession: ", userSession) + session.Store(dbKey, userSession) + } + } + } + + // if user session is still nil, return error + if (userSession == Session{}) || userSession.DB == nil { + log.Println("invalid dbKey") + err = errors.New("invalid dbKey") + return + } + return +} diff --git a/server/model.go b/server/model.go index b16ac22..d74e00f 100644 --- a/server/model.go +++ b/server/model.go @@ -7,7 +7,7 @@ type apiResponse struct { FileName string `json:"filename"` DBType string `json:"dbtype"` Message string `json:"message"` - Data Datagrid `json:"data"` + Data interface{} `json:"data"` Error []errorResponse `json:"error"` } diff --git a/server/server.go b/server/server.go index 455a8bb..1d0f9c8 100644 --- a/server/server.go +++ b/server/server.go @@ -54,9 +54,6 @@ func Serve(port string, debug bool) { // read the boltdb file dbroutes.GET("/", listKeyValue) - // read the boltdb file - dbroutes.GET("/{key}", getKeyValue) - // update the boltdb file dbroutes.PUT("/{key}", updateKeyValue) @@ -69,13 +66,13 @@ func Serve(port string, debug bool) { { // add new bucket from boltDB - bucketroutes.POST("/", insertBucket) + bucketroutes.POST("/", addBucket) // list all buckets from boltDB bucketroutes.GET("/", listBuckets) - // list all buckets from boltDB - bucketroutes.GET("/{bucket}", getBucket) + // set the bucket for the boltDB + bucketroutes.PUT("/{bucket}", setBucket) // remove a bucket from boltDB bucketroutes.DELETE("/", deleteBucket) From 43dc3d1565c6590e6a2005ed61c936a5abca6d80 Mon Sep 17 00:00:00 2001 From: Richie Varghese Date: Thu, 17 Feb 2022 00:38:46 +0530 Subject: [PATCH 2/4] replaced duplicate codeblocks --- server/bucket.go | 157 ++++++++++++--------------------- server/db.go | 191 +++++++++++------------------------------ server/manage_files.go | 179 +++++++------------------------------- server/model.go | 24 +++++- server/server.go | 89 +++++++++++++++++++ 5 files changed, 249 insertions(+), 391 deletions(-) diff --git a/server/bucket.go b/server/bucket.go index c40e249..0f6bcd9 100644 --- a/server/bucket.go +++ b/server/bucket.go @@ -12,78 +12,59 @@ import ( // Adds a new bucket to the open DB file. func addBucket(ctx *fasthttp.RequestCtx) { - // get the dbKey from params - dbKey := string(ctx.QueryArgs().Peek("dbKey")) - log.Println("dbKey:", dbKey) - - // load the db from user session - userSession, valid := session.Load(dbKey) - if !valid { - var err error - // try restoring user session - if userSession, err = restoreSession(dbKey); err != nil { - log.Println(err) - ctx.Error(err.Error(), fasthttp.StatusInternalServerError) - return - } + var bucket string + // get the dbKey from header + dbSession, err := sessionHandler(ctx) + if err != nil { + log.Println(err) + ctx.Error(err.Error(), fasthttp.StatusInternalServerError) + return } - sessionInfo := userSession.(Session) - switch sessionInfo.DBType { + switch dbSession.DBType { case database.BOLT_DB: - // get the DB type from params - bucket := string(ctx.QueryArgs().Peek("bucket")) + bucket = string(ctx.QueryArgs().Peek("bucket")) log.Println("bucket:", bucket) // add the bucket to the db - err := sessionInfo.DB.Conn().(*database.BoltDB).AddBucket(bucket) + err := dbSession.DB.Conn().(*database.BoltDB).AddBucket(bucket) if err != nil { log.Println(err) ctx.Error(err.Error(), fasthttp.StatusInternalServerError) return } + + default: + log.Println("DB type not supported:", dbSession.DBType) + ctx.Error("DB type not supported: "+dbSession.DBType, fasthttp.StatusInternalServerError) + return } // return success message to UI - json.NewEncoder(ctx).Encode(apiResponse{ - DBKey: dbKey, - FileName: sessionInfo.FileName, - DBType: sessionInfo.DBType, - Message: "Successfully added bucket to boltdb file: " + dbKey, - Error: nil, - }) + json.NewEncoder(ctx).Encode(generateResponse("Successfully added bucket: "+bucket, nil, &dbSession)) } // listBucket godoc - loads the list of buckets in a boltdb file func listBuckets(ctx *fasthttp.RequestCtx) { var buckets []string - // get the dbKey from params - dbKey := string(ctx.QueryArgs().Peek("dbKey")) - log.Println("dbKey:", dbKey) - // load the db from user session - userSession, valid := session.Load(dbKey) - if !valid { - var err error - // try restoring user session - if userSession, err = restoreSession(dbKey); err != nil { - log.Println(err) - ctx.Error(err.Error(), fasthttp.StatusInternalServerError) - return - } + // get the dbKey from header + dbSession, err := sessionHandler(ctx) + if err != nil { + log.Println(err) + ctx.Error(err.Error(), fasthttp.StatusInternalServerError) + return } - sessionInfo := userSession.(Session) - switch sessionInfo.DBType { + switch dbSession.DBType { case database.BOLT_DB: - var err error // get the list of buckets from the db - buckets, err = sessionInfo.DB.Conn().(*database.BoltDB).ListBuckets() + buckets, err = dbSession.DB.Conn().(*database.BoltDB).ListBuckets() if err != nil { log.Println(err) ctx.Error(err.Error(), fasthttp.StatusInternalServerError) @@ -92,100 +73,76 @@ func listBuckets(ctx *fasthttp.RequestCtx) { } // return success message to UI - json.NewEncoder(ctx).Encode(apiResponse{ - DBKey: dbKey, - FileName: sessionInfo.FileName, - DBType: sessionInfo.DBType, - Data: buckets, - Error: nil, - }) + json.NewEncoder(ctx).Encode(generateResponse("", buckets, &dbSession)) } // setBucket is the handler for the POST /api/v1/db/bucket/dbKey/file endpoint. func setBucket(ctx *fasthttp.RequestCtx) { - // get the dbKey from params - dbKey := string(ctx.QueryArgs().Peek("dbKey")) - log.Println("dbKey:", dbKey) - // load the db from user session - userSession, valid := session.Load(dbKey) - if !valid { - var err error - // try restoring user session - if userSession, err = restoreSession(dbKey); err != nil { - log.Println(err) - ctx.Error(err.Error(), fasthttp.StatusInternalServerError) - return - } + var bucket string + // get the dbKey from header + dbSession, err := sessionHandler(ctx) + if err != nil { + log.Println(err) + ctx.Error(err.Error(), fasthttp.StatusInternalServerError) + return } - sessionInfo := userSession.(Session) - switch sessionInfo.DBType { + switch dbSession.DBType { case database.BOLT_DB: - // get the DB type from params - bucket := string(ctx.UserValue("bucket").(string)) + bucket = string(ctx.UserValue("bucket").(string)) log.Println("bucket:", bucket) // set the bucket in the db - sessionInfo.DB.Conn().(*database.BoltDB).SetBucket(bucket) + dbSession.DB.Conn().(*database.BoltDB).SetBucket(bucket) + + default: + log.Println("DB type not supported:", dbSession.DBType) + ctx.Error("DB type not supported: "+dbSession.DBType, fasthttp.StatusInternalServerError) + return } // return success message to UI - json.NewEncoder(ctx).Encode(apiResponse{ - DBKey: dbKey, - FileName: sessionInfo.FileName, - DBType: sessionInfo.DBType, - Message: "Successfully applied default bucket: " + dbKey, - Error: nil, - }) + json.NewEncoder(ctx).Encode(generateResponse("Successfully applied default bucket: "+bucket, nil, &dbSession)) } // deleteBucket is the handler for the POST /api/v1/db/bucket/dbKey/file endpoint. // Removes a bucket from the open DB file. func deleteBucket(ctx *fasthttp.RequestCtx) { - // get the dbKey from params - dbKey := string(ctx.QueryArgs().Peek("dbKey")) - log.Println("dbKey:", dbKey) - - // load the db from user session - userSession, valid := session.Load(dbKey) - if !valid { - var err error - // try restoring user session - if userSession, err = restoreSession(dbKey); err != nil { - log.Println(err) - ctx.Error(err.Error(), fasthttp.StatusInternalServerError) - return - } + var bucket string + // get the dbKey from header + dbSession, err := sessionHandler(ctx) + if err != nil { + log.Println(err) + ctx.Error(err.Error(), fasthttp.StatusInternalServerError) + return } - sessionInfo := userSession.(Session) - switch sessionInfo.DBType { + switch dbSession.DBType { case database.BOLT_DB: // get the DB type from params - bucket := string(ctx.QueryArgs().Peek("bucket")) + bucket = string(ctx.QueryArgs().Peek("bucket")) log.Println("bucket:", bucket) // delete the bucket from the db - err := sessionInfo.DB.Conn().(*database.BoltDB).DeleteBucket(bucket) + err := dbSession.DB.Conn().(*database.BoltDB).DeleteBucket(bucket) if err != nil { log.Println(err) ctx.Error(err.Error(), fasthttp.StatusInternalServerError) return } + + default: + log.Println("DB type not supported:", dbSession.DBType) + ctx.Error("DB type not supported: "+dbSession.DBType, fasthttp.StatusInternalServerError) + return } // return success message to UI - json.NewEncoder(ctx).Encode(apiResponse{ - DBKey: dbKey, - FileName: sessionInfo.FileName, - DBType: sessionInfo.DBType, - Message: "Successfully removed bucket from boltdb file: " + dbKey, - Error: nil, - }) + json.NewEncoder(ctx).Encode(generateResponse("Successfully removed bucket: "+bucket, nil, &dbSession)) } diff --git a/server/db.go b/server/db.go index 647db7d..d3304c2 100644 --- a/server/db.go +++ b/server/db.go @@ -4,85 +4,43 @@ import ( "encoding/json" "fmt" "log" - "sync" "github.com/ric-v/divulge-keyvalue-db-ui/database" "github.com/valyala/fasthttp" ) -var session sync.Map - // listKeyValue is the handler for the POST /api/v1/bucket/ endpoint. // Opens the boltdb file and returns the file key-value paid for rendering in UI. func listKeyValue(ctx *fasthttp.RequestCtx) { var data []database.KeyValuePair - // get the dbKey from params - dbKey := string(ctx.QueryArgs().Peek("dbkey")) - log.Println("dbKey:", dbKey) - - // load the db from user session - userSession, valid := session.Load(dbKey) - if !valid { - var err error - // try restoring user session - if userSession, err = restoreSession(dbKey); err != nil { - log.Println(err) - ctx.Error(err.Error(), fasthttp.StatusInternalServerError) - return - } + + // get the dbKey from header + dbSession, err := sessionHandler(ctx) + if err != nil { + log.Println(err) + ctx.Error(err.Error(), fasthttp.StatusInternalServerError) + return } - sessionInfo := userSession.(Session) // switch on db type - switch sessionInfo.DBType { + switch dbSession.DBType { case database.BOLT_DB: - // get the DB type from params bucket := string(ctx.QueryArgs().Peek("bucket")) log.Println("bucket:", bucket) + } - // load the db from user session - userSession, valid := session.Load(dbKey) - if !valid { - log.Println("invalid dbKey") - ctx.Error("invalid dbKey", fasthttp.StatusBadRequest) - return - } - db := userSession.(Session).DB - - // open view on the boltdb file - views, err := db.List() - if err != nil { - log.Println(err) - ctx.Error(err.Error(), fasthttp.StatusInternalServerError) - return - } - log.Println("views:", views) - data = views - - case database.BUNT_DB: - - // load the db from user session - userSession, valid := session.Load(dbKey) - if !valid { - log.Println("invalid dbKey") - ctx.Error("invalid dbKey", fasthttp.StatusBadRequest) - return - } - db := userSession.(Session).DB - - // open view on the boltdb file - views, err := db.List() - if err != nil { - log.Println(err) - ctx.Error(err.Error(), fasthttp.StatusInternalServerError) - return - } - log.Println("views:", views) - data = views + // open view on the boltdb file + views, err := dbSession.DB.List() + if err != nil { + log.Println(err) + ctx.Error(err.Error(), fasthttp.StatusInternalServerError) + return } + log.Println("views:", views) + data = views // init new datagrid object var datagrid = Datagrid{ @@ -129,39 +87,23 @@ func listKeyValue(ctx *fasthttp.RequestCtx) { } // return success message to UI - json.NewEncoder(ctx).Encode(apiResponse{ - DBKey: dbKey, - FileName: sessionInfo.FileName, - DBType: sessionInfo.DBType, - Message: "Successfully opened boltdb file: " + dbKey, - Data: datagrid, - Error: nil, - }) + json.NewEncoder(ctx).Encode(generateResponse("", datagrid, &dbSession)) } // insertKeyValue is the handler for the POST /api/v1/db/dbKey/file endpoint. func insertKeyValue(ctx *fasthttp.RequestCtx) { - // get the dbKey from params - dbKey := string(ctx.QueryArgs().Peek("dbkey")) - log.Println("dbKey:", dbKey) - - // load the db from user session - userSession, valid := session.Load(dbKey) - if !valid { - var err error - // try restoring user session - if userSession, err = restoreSession(dbKey); err != nil { - log.Println(err) - ctx.Error(err.Error(), fasthttp.StatusInternalServerError) - return - } + // get the dbKey from header + dbSession, err := sessionHandler(ctx) + if err != nil { + log.Println(err) + ctx.Error(err.Error(), fasthttp.StatusInternalServerError) + return } - sessionInfo := userSession.(Session) // get the value from payload var data NewEntry - err := json.Unmarshal(ctx.PostBody(), &data) + err = json.Unmarshal(ctx.PostBody(), &data) if err != nil { log.Println(err) ctx.Error(err.Error(), fasthttp.StatusInternalServerError) @@ -169,7 +111,7 @@ func insertKeyValue(ctx *fasthttp.RequestCtx) { } // add new entry to DB - err = sessionInfo.DB.Add(data.Key, data.Value) + err = dbSession.DB.Add(data.Key, data.Value) if err != nil { log.Println(err) ctx.Error(err.Error(), fasthttp.StatusInternalServerError) @@ -177,13 +119,7 @@ func insertKeyValue(ctx *fasthttp.RequestCtx) { } // return success message to UI - json.NewEncoder(ctx).Encode(apiResponse{ - DBKey: dbKey, - FileName: sessionInfo.FileName, - DBType: sessionInfo.DBType, - Message: "Successfully added key-value pair to boltdb file: " + dbKey, - Error: nil, - }) + json.NewEncoder(ctx).Encode(generateResponse("Successfully added key-value pair", nil, &dbSession)) } // deleteKeyValue is the handler for the POST /api/v1/db/dbKey/file/key endpoint. @@ -194,26 +130,17 @@ func deleteKeyValue(ctx *fasthttp.RequestCtx) { Keys []string `json:"keys"` } - // get the dbKey from params - dbKey := string(ctx.QueryArgs().Peek("dbkey")) - log.Println("dbKey:", dbKey) - - // load the db from user session - userSession, valid := session.Load(dbKey) - if !valid { - var err error - // try restoring user session - if userSession, err = restoreSession(dbKey); err != nil { - log.Println(err) - ctx.Error(err.Error(), fasthttp.StatusInternalServerError) - return - } + // get the dbKey from header + dbSession, err := sessionHandler(ctx) + if err != nil { + log.Println(err) + ctx.Error(err.Error(), fasthttp.StatusInternalServerError) + return } - sessionInfo := userSession.(Session) // get the value from payload var keys deleteKeys - err := json.Unmarshal(ctx.PostBody(), &keys) + err = json.Unmarshal(ctx.PostBody(), &keys) if err != nil { log.Println(err) ctx.Error(err.Error(), fasthttp.StatusInternalServerError) @@ -221,21 +148,20 @@ func deleteKeyValue(ctx *fasthttp.RequestCtx) { } log.Println("keys:", keys) - switch sessionInfo.DBType { + switch dbSession.DBType { case database.BOLT_DB: // get the DB type from params bucket := string(ctx.QueryArgs().Peek("bucket")) log.Println("bucket:", bucket) - } // for each selected keys delete from DB for _, key := range keys.Keys { // delete key from DB - err = sessionInfo.DB.Delete(key) + err = dbSession.DB.Delete(key) if err != nil { log.Println(err) ctx.Error(err.Error(), fasthttp.StatusInternalServerError) @@ -244,43 +170,28 @@ func deleteKeyValue(ctx *fasthttp.RequestCtx) { } // return success message to UI - json.NewEncoder(ctx).Encode(apiResponse{ - DBKey: dbKey, - FileName: sessionInfo.FileName, - DBType: sessionInfo.DBType, - Message: "Successfully deleted keys from DB", - Error: nil, - }) + json.NewEncoder(ctx).Encode(generateResponse("Successfully deleted key-value pair", nil, &dbSession)) } // updateKeyValue is the handler for the POST /api/v1/db/dbKey/file/key endpoint. // Updates a key in the boltdb file. func updateKeyValue(ctx *fasthttp.RequestCtx) { - // get the dbKey from params - dbKey := string(ctx.QueryArgs().Peek("dbkey")) - log.Println("dbKey:", dbKey) + // get the dbKey from header + dbSession, err := sessionHandler(ctx) + if err != nil { + log.Println(err) + ctx.Error(err.Error(), fasthttp.StatusInternalServerError) + return + } // get the key from params key := string(ctx.UserValue("key").(string)) log.Println("key:", key) - // load the db from user session - userSession, valid := session.Load(dbKey) - if !valid { - var err error - // try restoring user session - if userSession, err = restoreSession(dbKey); err != nil { - log.Println(err) - ctx.Error(err.Error(), fasthttp.StatusInternalServerError) - return - } - } - sessionInfo := userSession.(Session) - // get the value from payload var data NewEntry - err := json.Unmarshal(ctx.PostBody(), &data) + err = json.Unmarshal(ctx.PostBody(), &data) if err != nil { log.Println(err) ctx.Error(err.Error(), fasthttp.StatusInternalServerError) @@ -288,7 +199,7 @@ func updateKeyValue(ctx *fasthttp.RequestCtx) { } log.Println("data:", data) - switch sessionInfo.DBType { + switch dbSession.DBType { case database.BOLT_DB: @@ -299,7 +210,7 @@ func updateKeyValue(ctx *fasthttp.RequestCtx) { } // add new entry to DB - err = sessionInfo.DB.Add(key, data.Value) + err = dbSession.DB.Add(key, data.Value) if err != nil { log.Println(err) ctx.Error(err.Error(), fasthttp.StatusInternalServerError) @@ -307,11 +218,5 @@ func updateKeyValue(ctx *fasthttp.RequestCtx) { } // return success message to UI - json.NewEncoder(ctx).Encode(apiResponse{ - DBKey: dbKey, - FileName: sessionInfo.FileName, - DBType: sessionInfo.DBType, - Message: "Successfully updated key: " + key, - Error: nil, - }) + json.NewEncoder(ctx).Encode(generateResponse("Successfully updated key-value pair", nil, &dbSession)) } diff --git a/server/manage_files.go b/server/manage_files.go index 32d5190..281820f 100644 --- a/server/manage_files.go +++ b/server/manage_files.go @@ -2,9 +2,6 @@ package server import ( "encoding/json" - "errors" - "io/fs" - "io/ioutil" "log" "os" @@ -57,17 +54,12 @@ func uploadFile(ctx *fasthttp.RequestCtx) { return } + dbSession := Session{dbKey, files.Filename, dbType, db} // store the db access in the session - session.Store(dbKey, Session{dbKey, files.Filename, dbType, db}) + session.Store(dbKey, dbSession) // return success message to UI - json.NewEncoder(ctx).Encode(apiResponse{ - DBKey: dbKey, - FileName: files.Filename, - DBType: dbType, - Message: "Successfully opened boltdb file", - Error: nil, - }) + json.NewEncoder(ctx).Encode(generateResponse("Successfully uploaded and verified db", nil, &dbSession)) } // newFile is the handler for the POST /api/v1/new endpoint. @@ -99,75 +91,49 @@ func newFile(ctx *fasthttp.RequestCtx) { return } + dbSession := Session{dbKey, file, dbType, db} // store the db access in the session - session.Store(dbKey, Session{dbKey, file, dbType, db}) + session.Store(dbKey, dbSession) // return success message to UI - json.NewEncoder(ctx).Encode(apiResponse{ - DBKey: dbKey, - FileName: file, - DBType: dbType, - Message: "Successfully created boltdb file: " + dbKey, - Error: nil, - }) + json.NewEncoder(ctx).Encode(generateResponse("Successfully created db", nil, &dbSession)) } // loadFile is the handler for the POST /api/v1/load endpoint. // Loads previously saved DB from local storage func loadFile(ctx *fasthttp.RequestCtx) { - // get the dbKey from params - dbKey := string(ctx.QueryArgs().Peek("dbkey")) - log.Println("dbKey:", dbKey) - - // load the db from user session - userSession, valid := session.Load(dbKey) - if !valid { - var err error - // try restoring user session - if userSession, err = restoreSession(dbKey); err != nil { - log.Println(err) - ctx.Error(err.Error(), fasthttp.StatusInternalServerError) - return - } + // get the dbKey from header + dbSession, err := sessionHandler(ctx) + if err != nil { + log.Println(err) + ctx.Error(err.Error(), fasthttp.StatusInternalServerError) + return } // return success message to UI - json.NewEncoder(ctx).Encode(apiResponse{ - DBKey: dbKey, - FileName: userSession.(Session).FileName, - DBType: userSession.(Session).DBType, - Message: "Successfully created boltdb file: " + dbKey, - Error: nil, - }) + json.NewEncoder(ctx).Encode(generateResponse("Reconnected to db", nil, &dbSession)) } // removeFile is the handler for the POST /api/v1/db/dbKey endpoint. func removeFile(ctx *fasthttp.RequestCtx) { - // get the dbKey from params - dbKey := string(ctx.QueryArgs().Peek("dbkey")) - log.Println("dbKey:", dbKey) - - // load the db from user session - userSession, valid := session.Load(dbKey) - if !valid { - var err error - // try restoring user session - if userSession, err = restoreSession(dbKey); err != nil { - log.Println(err) - ctx.Error(err.Error(), fasthttp.StatusInternalServerError) - return - } + // get the dbKey from header + dbSession, err := sessionHandler(ctx) + if err != nil { + log.Println(err) + ctx.Error(err.Error(), fasthttp.StatusInternalServerError) + return } - log.Println("userSession:", userSession) - userSession.(Session).DB.CloseDB() - dbType := userSession.(Session).DBType - session.Delete(dbKey) + + // close the db + dbSession.DB.CloseDB() + dbType := dbSession.DBType + session.Delete(dbSession.dbKey) // remove the folder - log.Println("removing folder:", "temp"+string(os.PathSeparator)+dbType+string(os.PathSeparator)+dbKey) - err := os.RemoveAll("temp" + string(os.PathSeparator) + dbType + string(os.PathSeparator) + dbKey) + log.Println("removing folder:", "temp"+string(os.PathSeparator)+dbType+string(os.PathSeparator)+dbSession.dbKey) + err = os.RemoveAll("temp" + string(os.PathSeparator) + dbType + string(os.PathSeparator) + dbSession.dbKey) if err != nil { log.Println(err) ctx.Error(err.Error(), fasthttp.StatusBadRequest) @@ -175,102 +141,21 @@ func removeFile(ctx *fasthttp.RequestCtx) { } // return success message to UI - json.NewEncoder(ctx).Encode(apiResponse{ - DBKey: dbKey, - Message: "Successfully closed boltdb file: " + dbKey, - Error: nil, - }) + json.NewEncoder(ctx).Encode(generateResponse("Cleared DB session and files", nil, &dbSession)) } // downloadFile is the handler for the GET /api/v1/db/download/dbKey/file endpoint. // Downloads the boltdb file to the UI. func downloadFile(ctx *fasthttp.RequestCtx) { - // get the dbKey from params - dbKey := string(ctx.QueryArgs().Peek("dbkey")) - log.Println("dbKey:", dbKey) - - // load the db from user session - userSession, valid := session.Load(dbKey) - if !valid { - var err error - // try restoring user session - if userSession, err = restoreSession(dbKey); err != nil { - log.Println(err) - ctx.Error(err.Error(), fasthttp.StatusInternalServerError) - return - } - } - sessionInfo := userSession.(Session) - sessionInfo.DB.CloseDB() - - // return the file to the UI - ctx.SendFile("temp" + string(os.PathSeparator) + sessionInfo.DBType + string(os.PathSeparator) + dbKey + string(os.PathSeparator) + sessionInfo.FileName) -} - -// restoreSession restores the user session from the boltdb / buntdb file if it exists on client -func restoreSession(dbKey string) (userSession Session, err error) { - - // get the file name under folder - dbTypes, err := ioutil.ReadDir("temp" + string(os.PathSeparator)) + // get the dbKey from header + dbSession, err := sessionHandler(ctx) if err != nil { log.Println(err) - err = errors.New("Error reading db folder: " + err.Error()) + ctx.Error(err.Error(), fasthttp.StatusInternalServerError) return } - log.Println("dbTypes:", dbTypes) - - // iterate over files - for _, dbType := range dbTypes { - - var dbKeys []fs.FileInfo - // get the file name under folder - dbKeys, err = ioutil.ReadDir("temp" + string(os.PathSeparator) + dbType.Name() + string(os.PathSeparator)) - if err != nil { - log.Println(err) - err = errors.New("Error reading db folder: " + err.Error()) - return - } - log.Println("dbKeys:", dbKeys) - - // iterate over files - for _, dbkey := range dbKeys { - log.Println("dbkey:", dbkey.Name(), " | dbKey:", dbkey) - - if dbkey.Name() == dbKey { - - var files []fs.FileInfo - // get the file name under folder - files, err = ioutil.ReadDir("temp" + string(os.PathSeparator) + dbType.Name() + string(os.PathSeparator) + dbKey) - if err != nil { - log.Println(err) - err = errors.New("Error reading db folder: " + err.Error()) - return - } - // get the file name - file := files[0].Name() - - userSession.DB, err = database.NewDB("temp"+string(os.PathSeparator)+dbType.Name()+string(os.PathSeparator)+dbKey+string(os.PathSeparator)+file, dbType.Name()) - if err != nil { - log.Println(err) - err = errors.New(err.Error()) - os.RemoveAll("temp" + string(os.PathSeparator) + dbKey) - return - } - - userSession = Session{dbKey, file, dbType.Name(), userSession.DB} - log.Println("userSession: ", userSession) - session.Store(dbKey, userSession) - } - } - } - - // if user session is still nil, return error - if (userSession == Session{}) || userSession.DB == nil { - log.Println("invalid dbKey") - err = errors.New("invalid dbKey") - return - } - return + // return the file to the UI + ctx.SendFile("temp" + string(os.PathSeparator) + dbSession.DBType + string(os.PathSeparator) + dbSession.dbKey + string(os.PathSeparator) + dbSession.FileName) } diff --git a/server/model.go b/server/model.go index d74e00f..c005969 100644 --- a/server/model.go +++ b/server/model.go @@ -1,6 +1,11 @@ package server -import "github.com/ric-v/divulge-keyvalue-db-ui/database" +import ( + "log" + "sync" + + "github.com/ric-v/divulge-keyvalue-db-ui/database" +) type apiResponse struct { DBKey string `json:"dbkey"` @@ -67,3 +72,20 @@ type NewEntry struct { Key string `json:"key"` Value string `json:"value"` } + +var session sync.Map + +// generateResponse godoc - generates a response for the UI +var generateResponse = func(msg string, data interface{}, dbSession *Session) (resp apiResponse) { + + // set the response message + resp = apiResponse{ + DBKey: dbSession.dbKey, + FileName: dbSession.FileName, + DBType: dbSession.DBType, + Message: msg, + Data: data, + } + log.Println("response : ", resp) + return +} diff --git a/server/server.go b/server/server.go index 1d0f9c8..54572fd 100644 --- a/server/server.go +++ b/server/server.go @@ -1,11 +1,15 @@ package server import ( + "errors" + "io/fs" + "io/ioutil" "log" "os" "path/filepath" "github.com/fasthttp/router" + "github.com/ric-v/divulge-keyvalue-db-ui/database" "github.com/valyala/fasthttp" ) @@ -102,3 +106,88 @@ func Serve(port string, debug bool) { log.Println("starting server on port:", port) log.Fatal(server.ListenAndServe(":" + port)) } + +// sessionHandler godoc - loads the db key from header for db access +func sessionHandler(ctx *fasthttp.RequestCtx) (dbSession Session, err error) { + + // get the dbKey from params + dbKey := string(ctx.QueryArgs().Peek("dbkey")) + log.Println("dbKey:", dbKey) + + // load the db from user session + userSession, valid := session.Load(dbKey) + if !valid { + + var dbTypes []fs.FileInfo + // get the file name under folder + dbTypes, err = ioutil.ReadDir("temp" + string(os.PathSeparator)) + if err != nil { + log.Println(err) + err = errors.New("Error reading db folder: " + err.Error()) + return + } + log.Println("dbTypes:", dbTypes) + + // iterate over files + for _, dbType := range dbTypes { + + var dbKeys []fs.FileInfo + // get the file name under folder + dbKeys, err = ioutil.ReadDir("temp" + string(os.PathSeparator) + dbType.Name() + string(os.PathSeparator)) + if err != nil { + log.Println(err) + err = errors.New("Error reading db folder: " + err.Error()) + return + } + log.Println("dbKeys:", dbKeys) + + // iterate over files + for _, dbkey := range dbKeys { + log.Println("dbkey:", dbkey.Name(), " | dbKey:", dbkey) + + if dbkey.Name() == dbKey { + + var files []fs.FileInfo + // get the file name under folder + files, err = ioutil.ReadDir("temp" + string(os.PathSeparator) + dbType.Name() + string(os.PathSeparator) + dbKey) + if err != nil { + log.Println(err) + err = errors.New("Error reading db folder: " + err.Error()) + return + } + + // get the file name + file := files[0].Name() + + var dbConn database.DB + dbConn, err = database.NewDB("temp"+string(os.PathSeparator)+dbType.Name()+string(os.PathSeparator)+dbKey+string(os.PathSeparator)+file, dbType.Name()) + if err != nil { + log.Println(err) + err = errors.New(err.Error()) + os.RemoveAll("temp" + string(os.PathSeparator) + dbKey) + return + } + + dbSession = Session{dbKey, file, dbType.Name(), dbConn} + log.Println("userSession: ", dbSession) + session.Store(dbKey, dbSession) + return + } + } + } + + // if user session is still nil, return error + if (dbSession == Session{}) || dbSession.DB == nil { + log.Println("invalid dbKey") + err = errors.New("invalid dbKey") + return + } + } + + var ok bool + if dbSession, ok = userSession.(Session); !ok { + err = errors.New("invalid session") + return + } + return +} From 4809c418ff096c94463110f18f1448f8630b8827 Mon Sep 17 00:00:00 2001 From: Richie Varghese Date: Mon, 21 Feb 2022 00:30:16 +0530 Subject: [PATCH 3/4] added bolt db bucket management --- database/boltdb_tx.go | 5 + server/bucket.go | 21 +- server/db.go | 8 +- server/manage_files.go | 6 +- server/server.go | 13 +- testfiles/boltdb_data.db | Bin 0 -> 32768 bytes ui/src/components/AddEntryModal.tsx | 6 +- ui/src/components/DBSelectStepper.tsx | 5 +- ui/src/components/DatagridComponents.tsx | 8 +- ui/src/components/ManageBucketsModal.tsx | 254 +++++++++++++++++++++++ ui/src/controllers/Controller.tsx | 32 ++- ui/src/controllers/DatagridList.tsx | 22 +- ui/src/controllers/FileUpload.tsx | 12 +- ui/src/controllers/NewFile.tsx | 6 +- ui/src/services/axios-common.tsx | 17 +- 15 files changed, 366 insertions(+), 49 deletions(-) create mode 100644 testfiles/boltdb_data.db create mode 100644 ui/src/components/ManageBucketsModal.tsx diff --git a/database/boltdb_tx.go b/database/boltdb_tx.go index dfba8d8..1fa93f3 100644 --- a/database/boltdb_tx.go +++ b/database/boltdb_tx.go @@ -125,6 +125,11 @@ func (db *BoltDB) SetBucket(bucketName string) { db.bucketName = []byte(bucketName) } +// SetBucket godoc - Sets the current bucket +func (db *BoltDB) GetDefBucket() string { + return string(db.bucketName) +} + // ListBuckets godoc - Lists all buckets in the database func (db *BoltDB) ListBuckets() (data []string, err error) { diff --git a/server/bucket.go b/server/bucket.go index 0f6bcd9..df8cbf5 100644 --- a/server/bucket.go +++ b/server/bucket.go @@ -3,6 +3,7 @@ package server import ( "encoding/json" "log" + "net/url" "github.com/ric-v/divulge-keyvalue-db-ui/database" "github.com/valyala/fasthttp" @@ -14,7 +15,7 @@ func addBucket(ctx *fasthttp.RequestCtx) { var bucket string // get the dbKey from header - dbSession, err := sessionHandler(ctx) + dbSession, err := handleDBSession(ctx) if err != nil { log.Println(err) ctx.Error(err.Error(), fasthttp.StatusInternalServerError) @@ -49,10 +50,15 @@ func addBucket(ctx *fasthttp.RequestCtx) { // listBucket godoc - loads the list of buckets in a boltdb file func listBuckets(ctx *fasthttp.RequestCtx) { - var buckets []string + type bucketList struct { + Buckets []string `json:"buckets"` + DefaultBucket string `json:"defaultBucket"` + } + + var buckets bucketList // get the dbKey from header - dbSession, err := sessionHandler(ctx) + dbSession, err := handleDBSession(ctx) if err != nil { log.Println(err) ctx.Error(err.Error(), fasthttp.StatusInternalServerError) @@ -64,12 +70,14 @@ func listBuckets(ctx *fasthttp.RequestCtx) { case database.BOLT_DB: var err error // get the list of buckets from the db - buckets, err = dbSession.DB.Conn().(*database.BoltDB).ListBuckets() + list, err := dbSession.DB.Conn().(*database.BoltDB).ListBuckets() if err != nil { log.Println(err) ctx.Error(err.Error(), fasthttp.StatusInternalServerError) return } + buckets.Buckets = list + buckets.DefaultBucket = dbSession.DB.Conn().(*database.BoltDB).GetDefBucket() } // return success message to UI @@ -81,7 +89,7 @@ func setBucket(ctx *fasthttp.RequestCtx) { var bucket string // get the dbKey from header - dbSession, err := sessionHandler(ctx) + dbSession, err := handleDBSession(ctx) if err != nil { log.Println(err) ctx.Error(err.Error(), fasthttp.StatusInternalServerError) @@ -93,6 +101,7 @@ func setBucket(ctx *fasthttp.RequestCtx) { case database.BOLT_DB: // get the DB type from params bucket = string(ctx.UserValue("bucket").(string)) + bucket, _ = url.QueryUnescape(bucket) log.Println("bucket:", bucket) // set the bucket in the db @@ -114,7 +123,7 @@ func deleteBucket(ctx *fasthttp.RequestCtx) { var bucket string // get the dbKey from header - dbSession, err := sessionHandler(ctx) + dbSession, err := handleDBSession(ctx) if err != nil { log.Println(err) ctx.Error(err.Error(), fasthttp.StatusInternalServerError) diff --git a/server/db.go b/server/db.go index d3304c2..8447c7d 100644 --- a/server/db.go +++ b/server/db.go @@ -16,7 +16,7 @@ func listKeyValue(ctx *fasthttp.RequestCtx) { var data []database.KeyValuePair // get the dbKey from header - dbSession, err := sessionHandler(ctx) + dbSession, err := handleDBSession(ctx) if err != nil { log.Println(err) ctx.Error(err.Error(), fasthttp.StatusInternalServerError) @@ -94,7 +94,7 @@ func listKeyValue(ctx *fasthttp.RequestCtx) { func insertKeyValue(ctx *fasthttp.RequestCtx) { // get the dbKey from header - dbSession, err := sessionHandler(ctx) + dbSession, err := handleDBSession(ctx) if err != nil { log.Println(err) ctx.Error(err.Error(), fasthttp.StatusInternalServerError) @@ -131,7 +131,7 @@ func deleteKeyValue(ctx *fasthttp.RequestCtx) { } // get the dbKey from header - dbSession, err := sessionHandler(ctx) + dbSession, err := handleDBSession(ctx) if err != nil { log.Println(err) ctx.Error(err.Error(), fasthttp.StatusInternalServerError) @@ -178,7 +178,7 @@ func deleteKeyValue(ctx *fasthttp.RequestCtx) { func updateKeyValue(ctx *fasthttp.RequestCtx) { // get the dbKey from header - dbSession, err := sessionHandler(ctx) + dbSession, err := handleDBSession(ctx) if err != nil { log.Println(err) ctx.Error(err.Error(), fasthttp.StatusInternalServerError) diff --git a/server/manage_files.go b/server/manage_files.go index 281820f..93cfa59 100644 --- a/server/manage_files.go +++ b/server/manage_files.go @@ -104,7 +104,7 @@ func newFile(ctx *fasthttp.RequestCtx) { func loadFile(ctx *fasthttp.RequestCtx) { // get the dbKey from header - dbSession, err := sessionHandler(ctx) + dbSession, err := handleDBSession(ctx) if err != nil { log.Println(err) ctx.Error(err.Error(), fasthttp.StatusInternalServerError) @@ -119,7 +119,7 @@ func loadFile(ctx *fasthttp.RequestCtx) { func removeFile(ctx *fasthttp.RequestCtx) { // get the dbKey from header - dbSession, err := sessionHandler(ctx) + dbSession, err := handleDBSession(ctx) if err != nil { log.Println(err) ctx.Error(err.Error(), fasthttp.StatusInternalServerError) @@ -149,7 +149,7 @@ func removeFile(ctx *fasthttp.RequestCtx) { func downloadFile(ctx *fasthttp.RequestCtx) { // get the dbKey from header - dbSession, err := sessionHandler(ctx) + dbSession, err := handleDBSession(ctx) if err != nil { log.Println(err) ctx.Error(err.Error(), fasthttp.StatusInternalServerError) diff --git a/server/server.go b/server/server.go index 54572fd..4e4c814 100644 --- a/server/server.go +++ b/server/server.go @@ -28,6 +28,13 @@ func Serve(port string, debug bool) { // create a new router r := router.New() r.HandleOPTIONS = true + r.GlobalOPTIONS = func(ctx *fasthttp.RequestCtx) { + // allow cors + ctx.Response.Header.Set("Access-Control-Allow-Origin", "*") + ctx.Response.Header.Set("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, OPTIONS") + ctx.Response.Header.Set("Access-Control-Allow-Headers", "*") + ctx.SetStatusCode(fasthttp.StatusOK) + } // /api/v1/ routes v1 := r.Group("/api/v1") @@ -107,11 +114,11 @@ func Serve(port string, debug bool) { log.Fatal(server.ListenAndServe(":" + port)) } -// sessionHandler godoc - loads the db key from header for db access -func sessionHandler(ctx *fasthttp.RequestCtx) (dbSession Session, err error) { +// handleDBSession godoc - loads the db key from header for db access +func handleDBSession(ctx *fasthttp.RequestCtx) (dbSession Session, err error) { // get the dbKey from params - dbKey := string(ctx.QueryArgs().Peek("dbkey")) + dbKey := string(ctx.Request.Header.Peek("Db-Key")) log.Println("dbKey:", dbKey) // load the db from user session diff --git a/testfiles/boltdb_data.db b/testfiles/boltdb_data.db new file mode 100644 index 0000000000000000000000000000000000000000..896ad888a1ba5cf1cd3789606f831583dd55b51a GIT binary patch literal 32768 zcmeI)J5Iwu5CG66d_;jnYG@G!q9C!j02Kuk904I$38@H@IVyJuJss=M2(X|*mT>V_ za&|TzuUD^Id*qbbw$t^S*MrTZj;G)M?@p)PF4*h(CO5~c;`sdK@%=6G^aKbHAV7cs z0RjXF5FkK+z<30bUHLjnW{5FkK+009C72oNAZ zU<3lOpYLAxk6=ZXK!5-N0t5&UAV7cs0RjXF3@s4v_v1Oh(DR#*009C72oNAZfB*pk z1PBoLe}SmupQN^uzy7Uj0rvB;%ExVWwSHKXPdWUtO;S8TNa-SH)bqu+0nU2klP}{% zS { value, }; - http - .post("/api/v1/db?dbkey=" + localStorage.getItem("dbkey"), payload) + http(String(localStorage.getItem("dbkey"))) + .post("/api/v1/db", payload) .then((resp) => { enqueueSnackbar("File created successfully", { key: "success", diff --git a/ui/src/components/DBSelectStepper.tsx b/ui/src/components/DBSelectStepper.tsx index bb826ba..41a924f 100644 --- a/ui/src/components/DBSelectStepper.tsx +++ b/ui/src/components/DBSelectStepper.tsx @@ -88,7 +88,6 @@ export default function VerticalLinearStepper( e.preventDefault(); setDbtype("boltdb"); }} - disabled /> @@ -107,7 +106,7 @@ export default function VerticalLinearStepper( inputProps={{ "aria-label": "controlled" }} /> } - label={checked ? "Create new database" : "Upload existing database"} + label={checked ? "Upload existing database" : "Create new database"} /> {checked ? ( @@ -115,10 +114,12 @@ export default function VerticalLinearStepper( setDbkey={setDbkey} dbName={dbName} setDbname={setDbname} + dbtype={dbtype} setStatus={setStatus} /> ) : ( { - http - .delete("/api/v1/db/?dbkey=" + dbkey, { data: { keys: keys } }) + http(dbkey) + .delete("/api/v1/db", { data: { keys: keys } }) .then((resp) => { enqueueSnackbar(resp.data.message, { key: "Deleted", @@ -199,7 +199,9 @@ export const CustomFooterStatusComponent = ({ onChange={(event, value) => apiRef.current.setPage(value - 1)} /> - *NOTE: Double click an entry to update the value + + *NOTE: Double click an entry to update the value + ); }; diff --git a/ui/src/components/ManageBucketsModal.tsx b/ui/src/components/ManageBucketsModal.tsx new file mode 100644 index 0000000..7cbfced --- /dev/null +++ b/ui/src/components/ManageBucketsModal.tsx @@ -0,0 +1,254 @@ +import { useState, useEffect } from "react"; +import { + Tooltip, + Fab, + Typography, + Dialog, + DialogTitle, + DialogContent, + DialogContentText, + TextField, + DialogActions, + Button, + IconButton, + List, + ListItemText, + ListItemButton, +} from "@mui/material"; +import http from "../services/axios-common"; +import AddIcon from "@material-ui/icons/Add"; +import DeleteIcon from "@mui/icons-material/Delete"; +import { useSnackbar } from "notistack"; +import { Box } from "@mui/system"; + +type Props = { + defBucket: string; + setBucket: React.Dispatch>; +}; + +const ManageBucketsModal = ({ defBucket, setBucket }: Props) => { + const [open, setOpen] = useState(false); + const [addBucket, setAddBucket] = useState(""); + const [buckets, setBuckets] = useState([]); + const { enqueueSnackbar, closeSnackbar } = useSnackbar(); + + // use effect to get buckets + useEffect(() => { + loadBuckets(); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []); + + // load buckets from server + const loadBuckets = () => { + http(String(localStorage.getItem("dbkey"))) + .get("/api/v1/bucket") + .then((resp) => { + setBuckets(resp.data.data.buckets); + setBucket(resp.data.data.defaultBucket); + setAddBucket(""); + }) + .catch((err) => { + enqueueSnackbar(err.response.data.message, { + key: "error", + variant: "error", + draggable: true, + onClick: () => { + closeSnackbar("error"); + }, + }); + }); + }; + + const handleClickOpen = () => { + setOpen(true); + }; + + const handleClose = () => { + setOpen(false); + setAddBucket(""); + }; + + // add bucket + const handleSubmit = () => { + console.log("bucket name: ", addBucket); + + http(String(localStorage.getItem("dbkey"))) + .post("/api/v1/bucket?bucket=" + addBucket) + .then((resp) => { + enqueueSnackbar("Bucket Added Successfully!", { + key: addBucket, + variant: "success", + draggable: true, + onClick: () => { + closeSnackbar(addBucket); + }, + }); + console.log(resp); + setAddBucket(""); + loadBuckets(); + }) + .catch((err) => { + enqueueSnackbar(err.response.data.message, { + key: "error", + variant: "error", + draggable: true, + onClick: () => { + closeSnackbar("error"); + }, + }); + }); + }; + + const handleSetBucket = (bucket: string) => { + http(String(localStorage.getItem("dbkey"))) + .put("/api/v1/bucket/" + bucket) + .then((resp) => { + enqueueSnackbar("Default Bucket Selected!", { + key: bucket, + variant: "success", + draggable: true, + onClick: () => { + closeSnackbar(bucket); + }, + }); + console.log(resp); + setAddBucket(""); + loadBuckets(); + }) + .catch((err) => { + enqueueSnackbar(err.response.data.message, { + key: "error", + variant: "error", + draggable: true, + onClick: () => { + closeSnackbar("error"); + }, + }); + }); + }; + + const handleDelete = (bucket: string) => { + http(String(localStorage.getItem("dbkey"))) + .delete("/api/v1/bucket?bucket=" + bucket) + .then((resp) => { + enqueueSnackbar("Bucket Removed Successfully!", { + key: bucket, + variant: "success", + draggable: true, + onClick: () => { + closeSnackbar(bucket); + }, + }); + console.log(resp); + setAddBucket(""); + loadBuckets(); + }) + .catch((err) => { + enqueueSnackbar(err.response.data.message, { + key: "error", + variant: "error", + draggable: true, + onClick: () => { + closeSnackbar("error"); + }, + }); + }); + }; + + return ( +
+ + handleClickOpen()} + > + {/* */} + Manage Buckets + + + + Manage Bolt DB Buckets + + + Add / Remove / Update and set Default bucket for the Bolt DB. + + + + Set default bucket + + {/* */} + + {buckets ? ( + buckets?.map((bucket) => ( + handleSetBucket(bucket)} + dense + sx={{ + "&:hover": { + backgroundColor: "primary.main", + color: "primary.contrastText", + }, + backgroundColor: + bucket === defBucket ? "secondary.main" : "", + color: + bucket === defBucket ? "secondary.contrastText" : "", + }} + > + + handleDelete(bucket)} + > + + + + )) + ) : ( + + No Buckets. Add buckets to continue. + + )} + + {/* */} + + + setAddBucket(e.target.value)} + /> + + + + + + + +
+ ); +}; + +export default ManageBucketsModal; diff --git a/ui/src/controllers/Controller.tsx b/ui/src/controllers/Controller.tsx index 653eae7..608c41c 100644 --- a/ui/src/controllers/Controller.tsx +++ b/ui/src/controllers/Controller.tsx @@ -7,6 +7,7 @@ import CloseIcon from "@material-ui/icons/Close"; import GetAppIcon from "@material-ui/icons/GetApp"; import http from "../services/axios-common"; import { useSnackbar } from "notistack"; +import ManageBucketsModal from "../components/ManageBucketsModal"; const Controller = () => { const [status, setStatus] = useState("connected"); @@ -14,6 +15,7 @@ const Controller = () => { const [dbkey, setDbkey] = useState(""); const [dbtype, setDbtype] = useState(""); const [loadView, setLoadView] = useState(false); + const [bucket, setBucket] = useState(""); const { enqueueSnackbar, closeSnackbar } = useSnackbar(); useEffect(() => { @@ -25,8 +27,8 @@ const Controller = () => { setDbkey(dbkey); setStatus("connected"); - http - .post("/api/v1/load?dbkey=" + dbkey) + http(dbkey) + .post("/api/v1/load") .then((resp) => { enqueueSnackbar("File loaded successfully", { key: "success", @@ -52,11 +54,11 @@ const Controller = () => { } } // eslint-disable-next-line react-hooks/exhaustive-deps - }, [dbkey, dbname, status, loadView, dbtype]); + }, [dbkey, bucket, loadView]); const downloadFile = (dbkey: string, dbname: string) => { - http - .get("/api/v1/download?dbkey=" + dbkey) + http(dbkey) + .get("/api/v1/download") .then((resp) => { const url = window.URL.createObjectURL(new Blob([resp.data])); const link = document.createElement("a"); @@ -82,8 +84,8 @@ const Controller = () => { setDbname: Dispatch>, setDbkey: Dispatch> ) => { - http - .delete("/api/v1/clear?dbkey=" + dbkey) + http(dbkey) + .delete("/api/v1/clear") .then((_resp) => { enqueueSnackbar("Database connection closed", { key: "success", @@ -141,6 +143,22 @@ const Controller = () => { Database:{" "} {dbname} + {dbtype === "boltdb" ? ( + + {bucket} + + ) : ( + "" + )} + + {dbtype === "boltdb" ? ( + + ) : ( + "" + )} >; }; -export default function FixedSizeGrid(props: Props) { +export default function FixedSizeGrid({ + dbkey, + dbname, + status, + setStatus, + setDbname, + setDbkey, + setLoadView, +}: Props) { const data = { columns: [], rows: [], @@ -34,8 +42,8 @@ export default function FixedSizeGrid(props: Props) { const [showDelete, setShowDelete] = useState(false); useEffect(() => { - http - .get("/api/v1/db/?dbkey=" + props.dbkey) + http(dbkey) + .get("/api/v1/db/") .then((resp) => { console.log("setting data", resp.data); setDataGrid(resp.data.data); @@ -59,8 +67,8 @@ export default function FixedSizeGrid(props: Props) { const newValue = e.value; const key: any = dataGrid.rows[+e.id - 1]; console.log("newValue", newValue, "key", key.key); - http - .put("/api/v1/db/" + key.key + "?dbkey=" + props.dbkey, { + http(dbkey) + .put("/api/v1/db/" + key.key, { value: newValue, key: key.key, }) @@ -115,9 +123,9 @@ export default function FixedSizeGrid(props: Props) { }} componentsProps={{ footer: { - status: props.status, + status: status, setUpdated: setUpdated, - dbkey: props.dbkey, + dbkey: dbkey, showDelete: showDelete, keys: keysToDelete, }, diff --git a/ui/src/controllers/FileUpload.tsx b/ui/src/controllers/FileUpload.tsx index d3a07cf..b04a3e5 100644 --- a/ui/src/controllers/FileUpload.tsx +++ b/ui/src/controllers/FileUpload.tsx @@ -5,12 +5,18 @@ import http from "../services/axios-common"; import ProgressBar from "../components/ProgressBar"; type FileUploadProps = { + dbtype: string; setDbname: React.Dispatch>; setDbkey: React.Dispatch>; setStatus: React.Dispatch>; }; -const FileUpload = ({ setDbname, setDbkey, setStatus }: FileUploadProps) => { +const FileUpload = ({ + dbtype, + setDbname, + setDbkey, + setStatus, +}: FileUploadProps) => { const { enqueueSnackbar, closeSnackbar } = useSnackbar(); const [file, setFile] = useState(""); @@ -25,8 +31,8 @@ const FileUpload = ({ setDbname, setDbkey, setStatus }: FileUploadProps) => { const formData = new FormData(); formData.append("file", file); - http - .post("/api/v1/upload?dbtype=buntdb", formData, { + http("") + .post("/api/v1/upload?dbtype=" + dbtype, formData, { headers: { "Content-Type": "multipart/form-data", }, diff --git a/ui/src/controllers/NewFile.tsx b/ui/src/controllers/NewFile.tsx index bdd642e..3b5bc13 100644 --- a/ui/src/controllers/NewFile.tsx +++ b/ui/src/controllers/NewFile.tsx @@ -7,6 +7,7 @@ type FileUploadProps = { dbName: string; setDbname: React.Dispatch>; setDbkey: React.Dispatch>; + dbtype: string; setStatus: React.Dispatch>; }; @@ -14,6 +15,7 @@ const NewFile = ({ dbName, setDbname, setDbkey, + dbtype, setStatus, }: FileUploadProps) => { const [inputDbName, setInputDbName] = useState(""); @@ -34,8 +36,8 @@ const NewFile = ({ }, [inputDbName, errorMessage]); const createNewFile = () => { - http - .post("/api/v1/new?file=" + dbName + ".db&dbtype=buntdb") + http("") + .post("/api/v1/new?file=" + dbName + ".db&dbtype=" + dbtype) .then((resp) => { enqueueSnackbar("File created successfully", { key: "success", diff --git a/ui/src/services/axios-common.tsx b/ui/src/services/axios-common.tsx index 87e8521..ff0cfc0 100644 --- a/ui/src/services/axios-common.tsx +++ b/ui/src/services/axios-common.tsx @@ -1,8 +1,13 @@ import axios from "axios"; -export default axios.create({ - baseURL: "", - headers: { - "Content-type": "application/json", - }, -}); +const http = (dbkey: String) => { + return axios.create({ + baseURL: "", + headers: { + "Content-type": "application/json", + "Db-Key": String(dbkey), + }, + }); +}; + +export default http; From a35cfdfa7826f7995d8a0ec709619824c4abc25b Mon Sep 17 00:00:00 2001 From: Richie Varghese Date: Mon, 21 Feb 2022 21:00:18 +0530 Subject: [PATCH 4/4] added buckets support for boltDB --- README.md | 4 +- server/db.go | 2 + server/manage_files.go | 69 +++++++++++++++++++++--- server/model.go | 4 +- ui/src/components/ManageBucketsModal.tsx | 8 +-- ui/src/controllers/Controller.tsx | 14 ++--- ui/src/controllers/DatagridList.tsx | 27 ++++++---- 7 files changed, 97 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index e8cd411..62bea3c 100644 --- a/README.md +++ b/README.md @@ -26,8 +26,8 @@ Simple DB CRUD operations service. Supports some golang Key-Value pair file base - [x] ~~Remove Key-Value pair~~ - [x] ~~Update Key-Value pair~~ - [x] ~~Download updated file~~ -- [ ] View Buckets in boltDB -- [ ] Add / remove bucket +- [x] View Buckets in boltDB +- [x] Add / remove bucket - [ ] Move/Copy Key-Value pair under a bucket to another bucket ## Usage diff --git a/server/db.go b/server/db.go index 8447c7d..4a3ed97 100644 --- a/server/db.go +++ b/server/db.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "log" + "net/url" "github.com/ric-v/divulge-keyvalue-db-ui/database" "github.com/valyala/fasthttp" @@ -187,6 +188,7 @@ func updateKeyValue(ctx *fasthttp.RequestCtx) { // get the key from params key := string(ctx.UserValue("key").(string)) + key, _ = url.QueryUnescape(key) log.Println("key:", key) // get the value from payload diff --git a/server/manage_files.go b/server/manage_files.go index 93cfa59..7908490 100644 --- a/server/manage_files.go +++ b/server/manage_files.go @@ -2,6 +2,8 @@ package server import ( "encoding/json" + "io" + "io/ioutil" "log" "os" @@ -44,8 +46,8 @@ func uploadFile(ctx *fasthttp.RequestCtx) { return } - // create the new boltdb file in the temp dir - log.Println("creating new boltdb file:", "temp"+string(os.PathSeparator)+dbType+string(os.PathSeparator)+dbKey+string(os.PathSeparator)+files.Filename) + // create the new db file in the temp dir + log.Println("creating new db file:", "temp"+string(os.PathSeparator)+dbType+string(os.PathSeparator)+dbKey+string(os.PathSeparator)+files.Filename) db, err := database.NewDB("temp"+string(os.PathSeparator)+dbType+string(os.PathSeparator)+dbKey+string(os.PathSeparator)+files.Filename, dbType) if err != nil { log.Println(err) @@ -129,11 +131,11 @@ func removeFile(ctx *fasthttp.RequestCtx) { // close the db dbSession.DB.CloseDB() dbType := dbSession.DBType - session.Delete(dbSession.dbKey) + session.Delete(dbSession.DbKey) // remove the folder - log.Println("removing folder:", "temp"+string(os.PathSeparator)+dbType+string(os.PathSeparator)+dbSession.dbKey) - err = os.RemoveAll("temp" + string(os.PathSeparator) + dbType + string(os.PathSeparator) + dbSession.dbKey) + log.Println("removing folder:", "temp"+string(os.PathSeparator)+dbType+string(os.PathSeparator)+dbSession.DbKey) + err = os.RemoveAll("temp" + string(os.PathSeparator) + dbType + string(os.PathSeparator) + dbSession.DbKey) if err != nil { log.Println(err) ctx.Error(err.Error(), fasthttp.StatusBadRequest) @@ -155,7 +157,62 @@ func downloadFile(ctx *fasthttp.RequestCtx) { ctx.Error(err.Error(), fasthttp.StatusInternalServerError) return } + var file = "temp" + string(os.PathSeparator) + dbSession.DBType + string(os.PathSeparator) + dbSession.DbKey + string(os.PathSeparator) + dbSession.FileName + + // check if db type is boltDB + if dbSession.DBType == database.BOLT_DB { + dbSession.DB.Conn().(*database.BoltDB).Sync() + dbSession.DB.CloseDB() + + data, err := ioutil.ReadFile(file) + if err != nil { + log.Println(err) + ctx.Error(err.Error(), fasthttp.StatusInternalServerError) + return + } + + ctx.SetContentType("application/octet-stream") + ctx.SetBody(data) + ctx.SetStatusCode(fasthttp.StatusOK) + return + + // // copy file to temp dir + // log.Println("copying file to temp dir:", "temp"+string(os.PathSeparator)+dbSession.DBType+string(os.PathSeparator)+dbSession.DbKey+string(os.PathSeparator)+dbSession.FileName) + // err = copyFile("temp"+string(os.PathSeparator)+dbSession.DBType+string(os.PathSeparator)+dbSession.DbKey+string(os.PathSeparator)+dbSession.FileName, "temp"+string(os.PathSeparator)+dbSession.FileName+".temp") + // if err != nil { + // log.Println(err) + // ctx.Error(err.Error(), fasthttp.StatusInternalServerError) + // return + // } + // file = "temp" + string(os.PathSeparator) + dbSession.FileName + ".temp" + } + log.Println("file:", file) // return the file to the UI - ctx.SendFile("temp" + string(os.PathSeparator) + dbSession.DBType + string(os.PathSeparator) + dbSession.dbKey + string(os.PathSeparator) + dbSession.FileName) + ctx.SendFile(file) +} + +// copyFile is a helper function to copy a file from one location to another. +func copyFile(src, dst string) error { + + // open source file + in, err := os.Open(src) + if err != nil { + return err + } + defer in.Close() + + // open destination file + out, err := os.Create(dst) + if err != nil { + return err + } + defer out.Close() + + // copy file + _, err = io.Copy(out, in) + if err != nil { + return err + } + return err } diff --git a/server/model.go b/server/model.go index c005969..033c8ce 100644 --- a/server/model.go +++ b/server/model.go @@ -61,7 +61,7 @@ type errorResponse struct { // Server - for managing db sessions with unique dbKey for each open session from UI type Session struct { - dbKey string + DbKey string FileName string DBType string DB database.DB @@ -80,7 +80,7 @@ var generateResponse = func(msg string, data interface{}, dbSession *Session) (r // set the response message resp = apiResponse{ - DBKey: dbSession.dbKey, + DBKey: dbSession.DbKey, FileName: dbSession.FileName, DBType: dbSession.DBType, Message: msg, diff --git a/ui/src/components/ManageBucketsModal.tsx b/ui/src/components/ManageBucketsModal.tsx index 7cbfced..65b07c9 100644 --- a/ui/src/components/ManageBucketsModal.tsx +++ b/ui/src/components/ManageBucketsModal.tsx @@ -16,10 +16,11 @@ import { ListItemButton, } from "@mui/material"; import http from "../services/axios-common"; -import AddIcon from "@material-ui/icons/Add"; import DeleteIcon from "@mui/icons-material/Delete"; import { useSnackbar } from "notistack"; import { Box } from "@mui/system"; +import LabelOutlinedIcon from "@mui/icons-material/LabelOutlined"; +import PlaylistAddCheckIcon from "@mui/icons-material/PlaylistAddCheck"; type Props = { defBucket: string; @@ -111,7 +112,6 @@ const ManageBucketsModal = ({ defBucket, setBucket }: Props) => { closeSnackbar(bucket); }, }); - console.log(resp); setAddBucket(""); loadBuckets(); }) @@ -163,7 +163,7 @@ const ManageBucketsModal = ({ defBucket, setBucket }: Props) => { color="secondary" onClick={() => handleClickOpen()} > - {/* */} + Manage Buckets @@ -193,6 +193,7 @@ const ManageBucketsModal = ({ defBucket, setBucket }: Props) => { buckets?.map((bucket) => ( handleSetBucket(bucket)} dense sx={{ @@ -206,6 +207,7 @@ const ManageBucketsModal = ({ defBucket, setBucket }: Props) => { bucket === defBucket ? "secondary.contrastText" : "", }} > + { closeSnackbar("success"); }, }); - console.log(resp); + console.log(resp.data); setDbname(resp.data.filename); setDbkey(resp.data.dbkey); + setDbtype(resp.data.dbtype); setStatus("connected"); setLoadView(true); localStorage.setItem("dbkey", resp.data.dbkey); @@ -60,7 +61,9 @@ const Controller = () => { http(dbkey) .get("/api/v1/download") .then((resp) => { - const url = window.URL.createObjectURL(new Blob([resp.data])); + const url = window.URL.createObjectURL( + new Blob([resp.data], { type: "application/file" }) + ); const link = document.createElement("a"); link.href = url; link.setAttribute("download", dbname); //or any other extension @@ -195,12 +198,9 @@ const Controller = () => { diff --git a/ui/src/controllers/DatagridList.tsx b/ui/src/controllers/DatagridList.tsx index 50dfbaf..59ff226 100644 --- a/ui/src/controllers/DatagridList.tsx +++ b/ui/src/controllers/DatagridList.tsx @@ -11,22 +11,16 @@ import { Box } from "@mui/material"; type Props = { dbkey: string; - dbname: string; status: string; - setStatus: React.Dispatch>; - setDbname: React.Dispatch>; - setDbkey: React.Dispatch>; - setLoadView: React.Dispatch>; + dbtype: string; + bucket: string; }; export default function FixedSizeGrid({ dbkey, - dbname, status, - setStatus, - setDbname, - setDbkey, - setLoadView, + dbtype, + bucket, }: Props) { const data = { columns: [], @@ -42,6 +36,17 @@ export default function FixedSizeGrid({ const [showDelete, setShowDelete] = useState(false); useEffect(() => { + if (dbtype === "boltdb" && bucket === "") { + enqueueSnackbar("Select a default bucket to load data from DB", { + key: "warning", + variant: "warning", + draggable: true, + onClick: () => { + closeSnackbar("warning"); + }, + }); + return; + } http(dbkey) .get("/api/v1/db/") .then((resp) => { @@ -61,7 +66,7 @@ export default function FixedSizeGrid({ }); setSelectionModel([]); // eslint-disable-next-line react-hooks/exhaustive-deps - }, [updated]); + }, [updated, dbtype, bucket]); const handleUpdate = (e: any) => { const newValue = e.value;