Skip to content

Commit

Permalink
Fix config for uploads when data server is not exposed (#936)
Browse files Browse the repository at this point in the history 
* Fix config for uploads when data server is not exposed

* Default to hostname, then localhost if data server is not specified

* Return a 404 if file isn't found in a GET request
  • Loading branch information
@ishank011
ishank011 committed Jul 6, 2020
1 parent 223eaed commit 9d637a6
Show file tree
Hide file tree
Showing 8 changed files with 42 additions and 22 deletions.
9 changes: 4 additions & 5 deletions cmd/reva/upload.go
View file
Expand Up @@ -139,7 +139,7 @@ func uploadCommand() *command {
return err
}

httpReq.Header.Set("X-Reva-Transfer", res.Token)
httpReq.Header.Set(datagateway.TokenTransportHeader, res.Token)
q := httpReq.URL.Query()
q.Add("xs", xs)
q.Add("xs_type", storageprovider.GRPC2PKGXS(xsType).String())
Expand Down Expand Up @@ -176,12 +176,11 @@ func uploadCommand() *command {
if err != nil {
return err
}
if token, ok := tokenpkg.ContextGetToken(ctx); ok {
c.Header.Add(tokenpkg.TokenHeader, token)
}
if res.Token != "" {
fmt.Printf("using X-Reva-Transfer header\n")
c.Header.Add(datagateway.TokenTransportHeader, res.Token)
} else if token, ok := tokenpkg.ContextGetToken(ctx); ok {
fmt.Printf("using %s header\n", tokenpkg.TokenHeader)
c.Header.Add(tokenpkg.TokenHeader, token)
}
tusc, err := tus.NewClient(dataServerURL, c)
if err != nil {
Expand Down
18 changes: 9 additions & 9 deletions docs/content/en/docs/config/grpc/services/storageprovider/_index.md
View file
Expand Up @@ -9,71 +9,71 @@ description: >
# _struct: config_

{{% dir name="mount_path" type="string" default="/" %}}
The path where the file system would be mounted. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L52)
The path where the file system would be mounted. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L51)
{{< highlight toml >}}
[grpc.services.storageprovider]
mount_path = "/"
{{< /highlight >}}
{{% /dir %}}

{{% dir name="mount_id" type="string" default="-" %}}
The ID of the mounted file system. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L53)
The ID of the mounted file system. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L52)
{{< highlight toml >}}
[grpc.services.storageprovider]
mount_id = "-"
{{< /highlight >}}
{{% /dir %}}

{{% dir name="driver" type="string" default="localhome" %}}
The storage driver to be used. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L54)
The storage driver to be used. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L53)
{{< highlight toml >}}
[grpc.services.storageprovider]
driver = "localhome"
{{< /highlight >}}
{{% /dir %}}

{{% dir name="drivers" type="map[string]map[string]interface{}" default="docs/config/packages/storage/fs" %}}
[[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L55)
[[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L54)
{{< highlight toml >}}
[grpc.services.storageprovider.drivers]
"[docs/config/packages/storage/fs]({{< ref "docs/config/packages/storage/fs" >}})"
{{< /highlight >}}
{{% /dir %}}

{{% dir name="tmp_folder" type="string" default="/var/tmp" %}}
Path to temporary folder. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L56)
Path to temporary folder. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L55)
{{< highlight toml >}}
[grpc.services.storageprovider]
tmp_folder = "/var/tmp"
{{< /highlight >}}
{{% /dir %}}

{{% dir name="data_server_url" type="string" default="http://localhost/data" %}}
The URL for the data server. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L57)
The URL for the data server. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L56)
{{< highlight toml >}}
[grpc.services.storageprovider]
data_server_url = "http://localhost/data"
{{< /highlight >}}
{{% /dir %}}

{{% dir name="expose_data_server" type="bool" default=false %}}
Whether to expose data server. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L58)
Whether to expose data server. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L57)
{{< highlight toml >}}
[grpc.services.storageprovider]
expose_data_server = false
{{< /highlight >}}
{{% /dir %}}

{{% dir name="disable_tus" type="bool" default=false %}}
Whether to disable TUS uploads. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L59)
Whether to disable TUS uploads. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L58)
{{< highlight toml >}}
[grpc.services.storageprovider]
disable_tus = false
{{< /highlight >}}
{{% /dir %}}

{{% dir name="available_checksums" type="map[string]uint32" default=nil %}}
List of available checksums. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L60)
List of available checksums. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L59)
{{< highlight toml >}}
[grpc.services.storageprovider]
available_checksums = nil
Expand Down
2 changes: 1 addition & 1 deletion docs/content/en/docs/config/http/services/dataprovider/_index.md
View file
Expand Up @@ -33,7 +33,7 @@ The configuration for the storage driver [[Ref]](https://github.com/cs3org/reva/
{{% /dir %}}

{{% dir name="disable_tus" type="bool" default=false %}}
Whether to disable TUS uploads. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/http/services/dataprovider/dataprovider.go#L42)
Whether to disable TUS uploads. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/http/services/dataprovider/dataprovider.go#L44)
{{< highlight toml >}}
[http.services.dataprovider]
disable_tus = false
Expand Down
1 change: 1 addition & 0 deletions examples/standalone/standalone.toml
View file
Expand Up @@ -12,6 +12,7 @@
[grpc.services.ocminvitemanager]
[grpc.services.ocmproviderauthorizer]

[http.services.datagateway]
[http.services.dataprovider]
[http.services.prometheus]
[http.services.ocmd]
10 changes: 8 additions & 2 deletions internal/grpc/services/storageprovider/storageprovider.go
View file
Expand Up @@ -35,7 +35,6 @@ import (
"github.com/cs3org/reva/pkg/errtypes"
"github.com/cs3org/reva/pkg/rgrpc"
"github.com/cs3org/reva/pkg/rgrpc/status"
"github.com/cs3org/reva/pkg/sharedconf"
"github.com/cs3org/reva/pkg/storage"
"github.com/cs3org/reva/pkg/storage/fs/registry"
"github.com/mitchellh/mapstructure"
Expand Down Expand Up @@ -77,7 +76,14 @@ func (c *config) init() {
c.TmpFolder = "/var/tmp/reva/tmp"
}

c.DataServerURL = sharedconf.GetDataGateway(c.DataServerURL)
if c.DataServerURL == "" {
host, err := os.Hostname()
if err != nil || host == "" {
c.DataServerURL = "http://0.0.0.0:19001/data"
} else {
c.DataServerURL = fmt.Sprintf("http://%s:19001/data", host)
}
}

// set sane defaults
if len(c.AvailableXS) == 0 {
Expand Down
2 changes: 1 addition & 1 deletion internal/http/services/datagateway/datagateway.go
View file
Expand Up @@ -60,7 +60,7 @@ type config struct {

func (c *config) init() {
if c.Prefix == "" {
c.Prefix = "data"
c.Prefix = "datagateway"
}

c.TransferSharedSecret = sharedconf.GetJWTSecret(c.TransferSharedSecret)
Expand Down
10 changes: 8 additions & 2 deletions internal/http/services/dataprovider/get.go
View file
Expand Up @@ -25,6 +25,7 @@ import (

provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
"github.com/cs3org/reva/pkg/appctx"
"github.com/cs3org/reva/pkg/errtypes"
)

func (s *svc) doGet(w http.ResponseWriter, r *http.Request) {
Expand All @@ -43,8 +44,13 @@ func (s *svc) doGet(w http.ResponseWriter, r *http.Request) {

rc, err := s.storage.Download(ctx, ref)
if err != nil {
log.Err(err).Msg("datasvc: error downloading file")
w.WriteHeader(http.StatusInternalServerError)
if _, ok := err.(errtypes.IsNotFound); ok {
log.Err(err).Msg("datasvc: file not found")
w.WriteHeader(http.StatusNotFound)
} else {
log.Err(err).Msg("datasvc: error downloading file")
w.WriteHeader(http.StatusInternalServerError)
}
return
}

Expand Down
12 changes: 10 additions & 2 deletions pkg/sharedconf/sharedconf.go
View file
Expand Up @@ -19,6 +19,9 @@
package sharedconf

import (
"fmt"
"os"

"github.com/mitchellh/mapstructure"
)

Expand All @@ -41,9 +44,14 @@ func Decode(v interface{}) error {
sharedConf.GatewaySVC = "0.0.0.0:19000"
}

// this is the default address we use for starting HTTP services
// this is the default address we use for the data gateway HTTP service
if sharedConf.DataGateway == "" {
sharedConf.DataGateway = "http://localhost:19001/data"
host, err := os.Hostname()
if err != nil || host == "" {
sharedConf.DataGateway = "http://0.0.0.0:19001/datagateway"
} else {
sharedConf.DataGateway = fmt.Sprintf("http://%s:19001/datagateway", host)
}
}

// TODO(labkode): would be cool to autogenerate one secret and print
Expand Down

0 comments on commit 9d637a6

Please sign in to comment.