From e74e2ed8ab2b9d26059c0713991f03d692ee3c03 Mon Sep 17 00:00:00 2001 From: Roman Perekhod Date: Tue, 12 Dec 2023 15:54:22 +0100 Subject: [PATCH] trash-bin cli has been exteneded by the list and restore commands --- changelog/unreleased/add-trach-bin-cli.md | 6 + services/gateway/pkg/config/config.go | 2 +- services/storage-users/README.md | 55 ++- .../storage-users/pkg/command/trash_bin.go | 454 ++++++++++++++++++ .../pkg/command/trash_bin_test.go | 65 +++ services/storage-users/pkg/config/config.go | 15 +- .../pkg/config/defaults/defaultconfig.go | 1 + 7 files changed, 587 insertions(+), 11 deletions(-) create mode 100644 changelog/unreleased/add-trach-bin-cli.md create mode 100644 services/storage-users/pkg/command/trash_bin_test.go diff --git a/changelog/unreleased/add-trach-bin-cli.md b/changelog/unreleased/add-trach-bin-cli.md new file mode 100644 index 00000000000..91485a28f0f --- /dev/null +++ b/changelog/unreleased/add-trach-bin-cli.md @@ -0,0 +1,6 @@ +Enhancement: Add cli commands for trash-binq + +We added the `list` and `restore` commands to the trash-bin items to the CLI + +https://github.com/owncloud/ocis/pull/7936 +https://github.com/owncloud/ocis/issues/7845 diff --git a/services/gateway/pkg/config/config.go b/services/gateway/pkg/config/config.go index 791064a67e4..f9262be035c 100644 --- a/services/gateway/pkg/config/config.go +++ b/services/gateway/pkg/config/config.go @@ -68,7 +68,7 @@ type Debug struct { } type GRPCConfig struct { - Addr string `yaml:"addr" env:"GATEWAY_GRPC_ADDR" desc:"The bind address of the GRPC service."` + Addr string `yaml:"addr" env:"OCIS_GATEWAY_GRPC_ADDR;GATEWAY_GRPC_ADDR" desc:"The bind address of the GRPC service."` TLS *shared.GRPCServiceTLS `yaml:"tls"` Namespace string `yaml:"-"` Protocol string `yaml:"protocol" env:"GATEWAY_GRPC_PROTOCOL" desc:"The transport protocol of the GRPC service."` diff --git a/services/storage-users/README.md b/services/storage-users/README.md index 1ceb861c580..e6e4ead0435 100644 --- a/services/storage-users/README.md +++ b/services/storage-users/README.md @@ -10,13 +10,13 @@ Starting with ocis version 3.0.0, the default backend for metadata switched to m Starting with Infinite Scale version 3.1, you can define a graceful shutdown period for the `storage-users` service. -IMPORTANT: The graceful shutdown period is only applicable if the `storage-users` service runs as standalone service. It does not apply if the `storage-users` service runs as part of the single binary or as single Docker environment. To build an environment where the `storage-users` service runs as a standalone service, you must start two instances, one _without_ the `storage-users` service and one _only with_ the the `storage-users` service. Note that both instances must be able to communicate on the same network. +IMPORTANT: The graceful shutdown period is only applicable if the `storage-users` service runs as standalone service. It does not apply if the `storage-users` service runs as part of the single binary or as single Docker environment. To build an environment where the `storage-users` service runs as a standalone service, you must start two instances, one _without_ the `storage-users` service and one _only with_ the the `storage-users` service. Note that both instances must be able to communicate on the same network. When hard-stopping Infinite Scale, for example with the `kill ` command (SIGKILL), it is possible and likely that not all data from the decomposedfs (metadata) has been written to the storage which may result in an inconsistent decomposedfs. When gracefully shutting down Infinite Scale, using a command like SIGTERM, the process will no longer accept any write requests from _other_ services and will try to write the internal open requests which can take an undefined duration based on many factors. To mitigate that situation, the following things have been implemented: * With the value of the environment variable `STORAGE_USERS_GRACEFUL_SHUTDOWN_TIMEOUT`, the `storage-users` service will delay its shutdown giving it time to finalize writing necessary data. This delay can be necessary if there is a lot of data to be saved and/or if storage access/thruput is slow. In such a case you would receive an error log entry informing you that not all data could be saved in time. To prevent such occurrences, you must increase the default value. -* If a shutdown error has been logged, the command-line maintenance tool [Inspect and Manipulate Node Metadata](https://doc.owncloud.com/ocis/next/maintenance/commands/commands.html#inspect-and-manipulate-node-metadata) can help to fix the issue. Please contact support for details. +* If a shutdown error has been logged, the command-line maintenance tool [Inspect and Manipulate Node Metadata](https://doc.owncloud.com/ocis/next/maintenance/commands/commands.html#inspect-and-manipulate-node-metadata) can help to fix the issue. Please contact support for details. ## CLI Commands @@ -37,7 +37,7 @@ When using Infinite Scale as user storage, a directory named `storage/users/uplo Example cases for expired uploads * When a user uploads a big file but the file exceeds the user-quota, the upload can't be moved to the target after it has finished. The file stays at the upload location until it is manually cleared. -* If the bandwidth is limited and the file to transfer can't be transferred completely before the upload expiration time is reached, the file expires and can't be processed. +* If the bandwidth is limited and the file to transfer can't be transferred completely before the upload expiration time is reached, the file expires and can't be processed. There are two commands available to manage unfinished uploads @@ -78,12 +78,13 @@ Cleaned uploads: -This command is about purging old trash-bin items of `project` spaces (spaces that have been created manually) and `personal` spaces. +This command is about the trash-bin to get an overview of items, restore items and purging old items of `project` spaces (spaces that have been created manually) and `personal` spaces. ```bash ocis storage-users trash-bin ``` +#### Purge-expired ```plaintext COMMANDS: purge-expired Purge all expired items from the trashbin @@ -97,6 +98,52 @@ The configuration for the `purge-expired` command is done by using the following * `STORAGE_USERS_PURGE_TRASH_BIN_PROJECT_DELETE_BEFORE` has a default value of `30 days`, which means the command will delete all files older than `30 days`. The value is human-readable, valid values are `24h`, `60m`, `60s` etc. `0` is equivalent to disable and prevents the deletion of `project space` trash-bin files. +#### List and Restore Trash-Bins Items + +To authenticate the cli command use `OCIS_MACHINE_AUTH_API_KEY=`. The `storage-users` cli tool uses the default address to establish the connection to the `gateway` service. If the connection is failed check your custom `gateway` +service `GATEWAY_GRPC_ADDR` configuration and set the same address to `storage-users` variable `OCIS_GATEWAY_GRPC_ADDR` or `STORAGE_USERS_GATEWAY_GRPC_ADDR`. + +The ID sources: +- 'userID' in a `https://{host}/graph/v1.0/me` +- personal 'spaceID' in a `https://{host}/graph/v1.0/me/drives?$filter=driveType+eq+personal` +- project 'spaceID' in a `https://{host}/graph/v1.0/me/drives?$filter=driveType+eq+project` + +```bash +NAME: + ocis storage-users trash-bin list - Print a list of all trash-bin items for a space. + +USAGE: + ocis storage-users trash-bin list command [command options] ['userID' required] ['spaceID' required] +``` + +```bash +NAME: + ocis storage-users trash-bin restore-all - Restore all trash-bin items for a space. + +USAGE: + ocis storage-users trash-bin restore-all command [command options] ['userID' required] ['spaceID' required] + +COMMANDS: + help, h Shows a list of commands or help for one command + +OPTIONS: + --option value, -o value The restore option defines the behavior for a file to be restored, where the file name already already exists in the target space. Supported values are: 'skip', 'replace' and 'keep-both'. The default value is 'skip' overwriting an existing file. +``` + +```bash +NAME: + ocis storage-users trash-bin restore - Restore a trash-bin item by ID. + +USAGE: + ocis storage-users trash-bin restore command [command options] ['userID' required] ['spaceID' required] ['itemID' required] + +COMMANDS: + help, h Shows a list of commands or help for one command + +OPTIONS: + --option value, -o value The restore option defines the behavior for a file to be restored, where the file name already already exists in the target space. Supported values are: 'skip', 'replace' and 'keep-both'. The default value is 'skip' overwriting an existing file. +``` + ## Caching The `storage-users` service caches stat, metadata and uuids of files and folders via the configured store in `STORAGE_USERS_STAT_CACHE_STORE`, `STORAGE_USERS_FILEMETADATA_CACHE_STORE` and `STORAGE_USERS_ID_CACHE_STORE`. Possible stores are: diff --git a/services/storage-users/pkg/command/trash_bin.go b/services/storage-users/pkg/command/trash_bin.go index 8489ff90bc2..fd02d304521 100644 --- a/services/storage-users/pkg/command/trash_bin.go +++ b/services/storage-users/pkg/command/trash_bin.go @@ -1,16 +1,50 @@ package command import ( + "context" + "fmt" + "path" + "path/filepath" + "strings" "time" + gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" + userv1beta1 "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/v2/pkg/events" + "github.com/cs3org/reva/v2/pkg/rgrpc/todo/pool" + "github.com/cs3org/reva/v2/pkg/storagespace" + "github.com/cs3org/reva/v2/pkg/utils" + "github.com/mohae/deepcopy" "github.com/owncloud/ocis/v2/ocis-pkg/config/configlog" + "github.com/owncloud/ocis/v2/ocis-pkg/tracing" "github.com/owncloud/ocis/v2/services/storage-users/pkg/config" "github.com/owncloud/ocis/v2/services/storage-users/pkg/config/parser" "github.com/owncloud/ocis/v2/services/storage-users/pkg/event" + "github.com/owncloud/ocis/v2/services/storage-users/pkg/logging" + "github.com/rs/zerolog/log" "github.com/urfave/cli/v2" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) +const ( + SKIP = iota + REPLACE + KEEP_BOTH + + retrievingErrorMsg = "trash-bin items retrieving error" +) + +var optionFlagTmpl = cli.StringFlag{ + Name: "option", + Value: "skip", + Aliases: []string{"o"}, + Usage: "The restore option defines the behavior for a file to be restored, where the file name already already exists in the target space. Supported values are: 'skip', 'replace' and 'keep-both'.", + DefaultText: "The default value is 'skip' overwriting an existing file", +} + // TrashBin wraps trash-bin related sub-commands. func TrashBin(cfg *config.Config) *cli.Command { return &cli.Command{ @@ -18,6 +52,9 @@ func TrashBin(cfg *config.Config) *cli.Command { Usage: "manage trash-bin's", Subcommands: []*cli.Command{ PurgeExpiredResources(cfg), + listTrashBinItems(cfg), + restoreAllTrashBinItems(cfg), + restoreTrashBindItem(cfg), }, } } @@ -53,3 +90,420 @@ func PurgeExpiredResources(cfg *config.Config) *cli.Command { }, } } + +func listTrashBinItems(cfg *config.Config) *cli.Command { + return &cli.Command{ + Name: "list", + Usage: "Print a list of all trash-bin items of a space.", + ArgsUsage: "['userID' required] ['spaceID' required]", + Flags: []cli.Flag{}, + Before: func(c *cli.Context) error { + return configlog.ReturnFatal(parser.ParseConfig(cfg)) + }, + Action: func(c *cli.Context) error { + log := logging.Configure(cfg.Service.Name, cfg.Log) + tp, err := tracing.GetServiceTraceProvider(cfg.Tracing, cfg.Service.Name) + if err != nil { + return err + } + var userID, spaceID string + if c.NArg() > 1 { + userID = c.Args().Get(0) + spaceID = c.Args().Get(1) + } + if userID == "" { + _ = cli.ShowSubcommandHelp(c) + return fmt.Errorf("userID is requered") + } + if spaceID == "" { + _ = cli.ShowSubcommandHelp(c) + return fmt.Errorf("spaceID is requered") + } + fmt.Printf("Getting trash-bin items for spaceID: '%s' ...\n", spaceID) + + ref, err := storagespace.ParseReference(spaceID) + if err != nil { + return err + } + client, err := pool.GetGatewayServiceClient(cfg.RevaGatewayGRPCAddr) + if err != nil { + log.Error().Err(err).Msg("error selecting next gateway client") + return err + } + ctx, _, err := utils.Impersonate(&userv1beta1.UserId{OpaqueId: userID}, client, cfg.MachineAuthAPIKey) + if err != nil { + log.Error().Err(err).Msg("could not impersonate") + return err + } + + spanOpts := []trace.SpanStartOption{ + trace.WithSpanKind(trace.SpanKindClient), + trace.WithAttributes( + attribute.KeyValue{Key: "userID", Value: attribute.StringValue(userID)}, + attribute.KeyValue{Key: "spaceID", Value: attribute.StringValue(spaceID)}, + ), + } + ctx, span := tp.Tracer("storage-users trash-bin list").Start(ctx, "serve static asset", spanOpts...) + defer span.End() + + res, err := client.ListRecycle(ctx, &provider.ListRecycleRequest{Ref: &ref, Key: "/"}) + if err != nil { + log.Error().Err(err).Msg(retrievingErrorMsg) + return err + } + if res.Status.Code != rpc.Code_CODE_OK { + return fmt.Errorf("%s %s", retrievingErrorMsg, res.Status.Code) + } + + if len(res.GetRecycleItems()) > 0 { + fmt.Println("The list of the trash-bin items. Use an itemID to restore.") + } else { + fmt.Println("The list is empty.") + } + + for _, item := range res.GetRecycleItems() { + fmt.Printf("itemID: '%s', path: '%s', type: '%s', delited at :%s\n", item.GetKey(), item.GetRef().GetPath(), itemType(item.GetType()), utils.TSToTime(item.GetDeletionTime()).UTC().Format(time.RFC3339)) + } + return nil + }, + } +} + +func restoreAllTrashBinItems(cfg *config.Config) *cli.Command { + var optionFlagVal string + var overwriteOption int + optionFlag := optionFlagTmpl + optionFlag.Destination = &optionFlagVal + return &cli.Command{ + Name: "restore-all", + Usage: "Restore all trash-bin items for a space.", + ArgsUsage: "['userID' required] ['spaceID' required]", + Flags: []cli.Flag{ + &optionFlag, + }, + Before: func(c *cli.Context) error { + return configlog.ReturnFatal(parser.ParseConfig(cfg)) + }, + Action: func(c *cli.Context) error { + log := logging.Configure(cfg.Service.Name, cfg.Log) + tp, err := tracing.GetServiceTraceProvider(cfg.Tracing, cfg.Service.Name) + if err != nil { + return err + } + c.Lineage() + var userID, spaceID string + if c.NArg() > 1 { + userID = c.Args().Get(0) + spaceID = c.Args().Get(1) + } + if userID == "" { + _ = cli.ShowSubcommandHelp(c) + return cli.Exit("The userID is required", 1) + } + if spaceID == "" { + _ = cli.ShowSubcommandHelp(c) + return cli.Exit("The spaceID is required", 1) + } + switch optionFlagVal { + case "skip": + overwriteOption = SKIP + case "replace": + overwriteOption = REPLACE + case "keep-both": + overwriteOption = KEEP_BOTH + default: + _ = cli.ShowSubcommandHelp(c) + return cli.Exit("The option flag is invalid", 1) + } + fmt.Printf("Restoring trash-bin items for spaceID: '%s' ...\n", spaceID) + ref, err := storagespace.ParseReference(spaceID) + if err != nil { + return err + } + client, err := pool.GetGatewayServiceClient(cfg.RevaGatewayGRPCAddr) + if err != nil { + log.Error().Err(err).Msg("error selecting next gateway client") + return err + } + ctx, _, err := utils.Impersonate(&userv1beta1.UserId{OpaqueId: userID}, client, cfg.MachineAuthAPIKey) + if err != nil { + log.Error().Err(err).Msg("could not impersonate") + return err + } + + spanOpts := []trace.SpanStartOption{ + trace.WithSpanKind(trace.SpanKindClient), + trace.WithAttributes( + attribute.KeyValue{Key: "option", Value: attribute.StringValue(optionFlagVal)}, + attribute.KeyValue{Key: "userID", Value: attribute.StringValue(userID)}, + attribute.KeyValue{Key: "spaceID", Value: attribute.StringValue(spaceID)}, + ), + } + ctx, span := tp.Tracer("storage-users trash-bin restore-all").Start(ctx, "serve static asset", spanOpts...) + defer span.End() + + res, err := client.ListRecycle(ctx, &provider.ListRecycleRequest{Ref: &ref, Key: "/"}) + if err != nil { + log.Error().Err(err).Msg(retrievingErrorMsg) + return err + } + if res.Status.Code != rpc.Code_CODE_OK { + return fmt.Errorf("%s %s", retrievingErrorMsg, res.Status.Code) + } + if len(res.GetRecycleItems()) == 0 { + return cli.Exit("The trash-bin is empty. Nothing to restore", 0) + } + + for { + fmt.Printf("Foud %d items that could be restored, continue (Y/n), show the items list (s): ", len(res.GetRecycleItems())) + var i string + _, err := fmt.Scanf("%s", &i) + if err != nil { + log.Err(err).Send() + continue + } + if strings.ToLower(i) == "y" { + break + } else if strings.ToLower(i) == "n" { + return nil + } else if strings.ToLower(i) == "s" { + for _, item := range res.GetRecycleItems() { + fmt.Printf("itemID: '%s', path: '%s', type: '%s', delited at: %s\n", item.GetKey(), item.GetRef().GetPath(), itemType(item.GetType()), utils.TSToTime(item.GetDeletionTime()).UTC().Format(time.RFC3339)) + } + } + } + + fmt.Printf("\nRun restoring-all with option=%s\n", optionFlagVal) + for _, item := range res.GetRecycleItems() { + fmt.Printf("restoring itemID: '%s', path: '%s', type: '%s'\n", item.GetKey(), item.GetRef().GetPath(), itemType(item.GetType())) + dstRes, err := restore(ctx, client, ref, item, overwriteOption) + if err != nil { + return err + } + fmt.Printf("itemID: '%s', path: '%s', restored as '%s'\n", item.GetKey(), item.GetRef().GetPath(), dstRes.GetPath()) + } + return nil + }, + } +} + +func restoreTrashBindItem(cfg *config.Config) *cli.Command { + var optionFlagVal string + var overwriteOption int + optionFlag := optionFlagTmpl + optionFlag.Destination = &optionFlagVal + return &cli.Command{ + Name: "restore", + Usage: "Restore a trash-bin item by ID.", + ArgsUsage: "['userId' required] ['spaceID' required] ['itemID' required]", + Flags: []cli.Flag{ + &optionFlag, + }, + Before: func(c *cli.Context) error { + return configlog.ReturnFatal(parser.ParseConfig(cfg)) + }, + Action: func(c *cli.Context) error { + log := logging.Configure(cfg.Service.Name, cfg.Log) + tp, err := tracing.GetServiceTraceProvider(cfg.Tracing, cfg.Service.Name) + if err != nil { + return err + } + c.Lineage() + var userID, spaceID, itemID string + if c.NArg() > 2 { + userID = c.Args().Get(0) + spaceID = c.Args().Get(1) + itemID = c.Args().Get(2) + } + if userID == "" { + _ = cli.ShowSubcommandHelp(c) + return fmt.Errorf("userID is requered") + } + if spaceID == "" { + _ = cli.ShowSubcommandHelp(c) + return fmt.Errorf("spaceID is requered") + } + if itemID == "" { + _ = cli.ShowSubcommandHelp(c) + return fmt.Errorf("itemID is requered") + } + switch optionFlagVal { + case "skip": + overwriteOption = SKIP + case "replace": + overwriteOption = REPLACE + case "keep-both": + overwriteOption = KEEP_BOTH + default: + _ = cli.ShowSubcommandHelp(c) + return cli.Exit("The option flag is invalid", 1) + } + + ref, err := storagespace.ParseReference(spaceID) + if err != nil { + return err + } + client, err := pool.GetGatewayServiceClient(cfg.RevaGatewayGRPCAddr) + if err != nil { + log.Error().Err(err).Msg("error selecting gateway client") + return err + } + ctx, _, err := utils.Impersonate(&userv1beta1.UserId{OpaqueId: userID}, client, cfg.MachineAuthAPIKey) + if err != nil { + log.Error().Err(err).Msg("could not impersonate") + return err + } + + spanOpts := []trace.SpanStartOption{ + trace.WithSpanKind(trace.SpanKindClient), + trace.WithAttributes( + attribute.KeyValue{Key: "option", Value: attribute.StringValue(optionFlagVal)}, + attribute.KeyValue{Key: "userID", Value: attribute.StringValue(userID)}, + attribute.KeyValue{Key: "spaceID", Value: attribute.StringValue(spaceID)}, + attribute.KeyValue{Key: "itemID", Value: attribute.StringValue(itemID)}, + ), + } + ctx, span := tp.Tracer("storage-users trash-bin restore").Start(ctx, "serve static asset", spanOpts...) + defer span.End() + + res, err := client.ListRecycle(ctx, &provider.ListRecycleRequest{Ref: &ref, Key: "/"}) + if err != nil { + log.Error().Err(err).Msg(retrievingErrorMsg) + return err + } + if res.Status.Code != rpc.Code_CODE_OK { + return fmt.Errorf("%s %s", retrievingErrorMsg, res.Status.Code) + } + + var found bool + var itemRef *provider.RecycleItem + for _, item := range res.GetRecycleItems() { + if item.GetKey() == itemID { + itemRef = item + found = true + break + } + } + if !found { + return fmt.Errorf("itemID '%s' not found", itemID) + } + fmt.Printf("\nRun restoring with option=%s\n", optionFlagVal) + fmt.Printf("restoring itemID: '%s', path: '%s', type: '%s'\n", itemRef.GetKey(), itemRef.GetRef().GetPath(), itemType(itemRef.GetType())) + dstRes, err := restore(ctx, client, ref, itemRef, overwriteOption) + if err != nil { + return err + } + fmt.Printf("itemID: '%s', path: '%s', restored as '%s'\n", itemRef.GetKey(), itemRef.GetRef().GetPath(), dstRes.GetPath()) + return nil + }, + } +} + +func restore(ctx context.Context, client gateway.GatewayAPIClient, ref provider.Reference, item *provider.RecycleItem, overwriteOption int) (*provider.Reference, error) { + dst, _ := deepcopy.Copy(ref).(provider.Reference) + dst.Path = utils.MakeRelativePath(item.GetRef().GetPath()) + // Restore request + req := &provider.RestoreRecycleItemRequest{ + Ref: &ref, + Key: path.Join(item.GetKey(), "/"), + RestoreRef: &dst, + } + + exists, dstStatRes, err := isDestinationExists(ctx, client, dst) + if err != nil { + return &dst, err + } + + if exists { + fmt.Printf("destination '%s' exists.\n", dstStatRes.GetInfo().GetPath()) + switch overwriteOption { + case SKIP: + return &dst, nil + case REPLACE: + // delete existing tree + delReq := &provider.DeleteRequest{Ref: &dst} + delRes, err := client.Delete(ctx, delReq) + if err != nil { + return &dst, fmt.Errorf("error sending grpc delete request %w", err) + } + if delRes.Status.Code != rpc.Code_CODE_OK && delRes.Status.Code != rpc.Code_CODE_NOT_FOUND { + return &dst, fmt.Errorf("deleting error %w", err) + } + case KEEP_BOTH: + // modify the file name + req.RestoreRef, err = resolveDestination(ctx, client, dst) + if err != nil { + return &dst, fmt.Errorf("trash-bin item restoring error %w", err) + } + } + } + + res, err := client.RestoreRecycleItem(ctx, req) + if err != nil { + log.Error().Err(err).Msg("trash-bin item restoring error") + return req.RestoreRef, err + } + if res.Status.Code != rpc.Code_CODE_OK { + return req.RestoreRef, fmt.Errorf("trash-bin item restoring error %s", res.Status.Code) + } + return req.RestoreRef, nil +} + +func resolveDestination(ctx context.Context, client gateway.GatewayAPIClient, dstRef provider.Reference) (*provider.Reference, error) { + dst := dstRef + for i := 1; i < 100; i++ { + dst.Path = modifyFilename(dstRef.Path, i) + exists, _, err := isDestinationExists(ctx, client, dst) + if err != nil { + return nil, err + } + if exists { + continue + } + return &dst, nil + } + return nil, fmt.Errorf("too many attempts to resolve the destination") +} + +func isDestinationExists(ctx context.Context, client gateway.GatewayAPIClient, dst provider.Reference) (bool, *provider.StatResponse, error) { + dstStatReq := &provider.StatRequest{Ref: &dst} + dstStatRes, err := client.Stat(ctx, dstStatReq) + if err != nil { + return false, nil, fmt.Errorf("error sending grpc stat request %w", err) + } + if dstStatRes.GetStatus().GetCode() == rpc.Code_CODE_OK { + return true, dstStatRes, nil + } + if dstStatRes.GetStatus().GetCode() == rpc.Code_CODE_NOT_FOUND { + return false, dstStatRes, nil + } + return false, dstStatRes, fmt.Errorf("stat request failed %s", dstStatRes.GetStatus()) +} + +// modify the file name like UI do +func modifyFilename(filename string, mod int) string { + var extension string + var found bool + expected := []string{".tar.gz", ".tar.bz", ".tar.bz2"} + for _, s := range expected { + var prefix string + prefix, found = strings.CutSuffix(strings.ToLower(filename), s) + if found { + extension = strings.TrimPrefix(filename, prefix) + break + } + } + if !found { + extension = filepath.Ext(filename) + } + name := filename[0 : len(filename)-len(extension)] + return fmt.Sprintf("%s (%d)%s", name, mod, extension) +} + +func itemType(it provider.ResourceType) string { + var itemType = "file" + if it == provider.ResourceType_RESOURCE_TYPE_CONTAINER { + itemType = "folder" + } + return itemType +} diff --git a/services/storage-users/pkg/command/trash_bin_test.go b/services/storage-users/pkg/command/trash_bin_test.go new file mode 100644 index 00000000000..26cfd4bc156 --- /dev/null +++ b/services/storage-users/pkg/command/trash_bin_test.go @@ -0,0 +1,65 @@ +package command + +import ( + "testing" +) + +func Test_modifyFilename(t *testing.T) { + type args struct { + filename string + mod int + } + tests := []struct { + name string + args args + want string + }{ + { + name: "file", + args: args{filename: "file.txt", mod: 1}, + want: "file (1).txt", + }, + { + name: "file with path", + args: args{filename: "./file.txt", mod: 1}, + want: "./file (1).txt", + }, + { + name: "file with path 2", + args: args{filename: "./subdir/file.tar.gz", mod: 99}, + want: "./subdir/file (99).tar.gz", + }, + { + name: "file with path 3", + args: args{filename: "./sub dir/new file.tar.gz", mod: 99}, + want: "./sub dir/new file (99).tar.gz", + }, + { + name: "file without ext", + args: args{filename: "./subdir/file", mod: 2}, + want: "./subdir/file (2)", + }, + { + name: "file without ext 2", + args: args{filename: "./subdir/file 1", mod: 2}, + want: "./subdir/file 1 (2)", + }, + { + name: "file with emoji", + args: args{filename: "./subdir/file 🙂.tar.gz", mod: 3}, + want: "./subdir/file 🙂 (3).tar.gz", + }, + { + name: "file with emoji 2", + args: args{filename: "./subdir/file 🙂", mod: 2}, + want: "./subdir/file 🙂 (2)", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := modifyFilename(tt.args.filename, tt.args.mod); got != tt.want { + t.Errorf("modifyFilename() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/services/storage-users/pkg/config/config.go b/services/storage-users/pkg/config/config.go index 5423259a0f4..3d10974a9fe 100644 --- a/services/storage-users/pkg/config/config.go +++ b/services/storage-users/pkg/config/config.go @@ -18,16 +18,19 @@ type Config struct { GRPC GRPCConfig `yaml:"grpc"` HTTP HTTPConfig `yaml:"http"` - TokenManager *TokenManager `yaml:"token_manager"` - Reva *shared.Reva `yaml:"reva"` + TokenManager *TokenManager `yaml:"token_manager"` + Reva *shared.Reva `yaml:"reva"` + RevaGatewayGRPCAddr string `yaml:"gateway_addr" env:"OCIS_GATEWAY_GRPC_ADDR;STORAGE_USERS_GATEWAY_GRPC_ADDR" desc:"The bind address of the gateway GRPC address."` + MachineAuthAPIKey string `yaml:"machine_auth_api_key" env:"OCIS_MACHINE_AUTH_API_KEY" desc:"Machine auth API key used to validate internal requests necessary for the access to resources from other services."` SkipUserGroupsInToken bool `yaml:"skip_user_groups_in_token" env:"STORAGE_USERS_SKIP_USER_GROUPS_IN_TOKEN" desc:"Disables the loading of user's group memberships from the reva access token."` GracefulShutdownTimeout int `yaml:"graceful_shutdown_timeout" env:"STORAGE_USERS_GRACEFUL_SHUTDOWN_TIMEOUT" desc:"The number of seconds to wait for the 'storage-users' service to shutdown cleanly before exiting with an error that gets logged. Note: This setting is only applicable when running the 'storage-users' service as a standalone service. See the text description for more details."` - Driver string `yaml:"driver" env:"STORAGE_USERS_DRIVER" desc:"The storage driver which should be used by the service. Defaults to 'ocis', Supported values are: 'ocis', 's3ng' and 'owncloudsql'. The 'ocis' driver stores all data (blob and meta data) in an POSIX compliant volume. The 's3ng' driver stores metadata in a POSIX compliant volume and uploads blobs to the s3 bucket."` - Drivers Drivers `yaml:"drivers"` - DataServerURL string `yaml:"data_server_url" env:"STORAGE_USERS_DATA_SERVER_URL" desc:"URL of the data server, needs to be reachable by the data gateway provided by the frontend service or the user if directly exposed."` - DataGatewayURL string `yaml:"data_gateway_url" env:"STORAGE_USERS_DATA_GATEWAY_URL" desc:"URL of the data gateway server"` + Driver string `yaml:"driver" env:"STORAGE_USERS_DRIVER" desc:"The storage driver which should be used by the service. Defaults to 'ocis', Supported values are: 'ocis', 's3ng' and 'owncloudsql'. The 'ocis' driver stores all data (blob and meta data) in an POSIX compliant volume. The 's3ng' driver stores metadata in a POSIX compliant volume and uploads blobs to the s3 bucket."` + Drivers Drivers `yaml:"drivers"` + DataServerURL string `yaml:"data_server_url" env:"STORAGE_USERS_DATA_SERVER_URL" desc:"URL of the data server, needs to be reachable by the data gateway provided by the frontend service or the user if directly exposed."` + DataGatewayURL string `yaml:"data_gateway_url" env:"STORAGE_USERS_DATA_GATEWAY_URL" desc:"URL of the data gateway server"` + TransferExpires int64 `yaml:"transfer_expires" env:"STORAGE_USERS_TRANSFER_EXPIRES" desc:"the time after which the token for upload postprocessing expires"` Events Events `yaml:"events"` StatCache StatCache `yaml:"stat_cache"` diff --git a/services/storage-users/pkg/config/defaults/defaultconfig.go b/services/storage-users/pkg/config/defaults/defaultconfig.go index 9cb71e4f26e..aeafdd70a21 100644 --- a/services/storage-users/pkg/config/defaults/defaultconfig.go +++ b/services/storage-users/pkg/config/defaults/defaultconfig.go @@ -44,6 +44,7 @@ func DefaultConfig() *config.Config { Reva: shared.DefaultRevaConfig(), DataServerURL: "http://localhost:9158/data", DataGatewayURL: "https://localhost:9200/data", + RevaGatewayGRPCAddr: "127.0.0.1:9142", TransferExpires: 86400, UploadExpiration: 24 * 60 * 60, GracefulShutdownTimeout: 30,