Skip to content

Commit

Permalink
chore: Sync with upstream Thanos (#13)
Browse files Browse the repository at this point in the history
Sync with upstream Thanos

Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com>
  • Loading branch information
kakkoyun committed Jul 15, 2022
1 parent 2f10012 commit ce33880
Show file tree
Hide file tree
Showing 20 changed files with 420 additions and 86 deletions.
8 changes: 4 additions & 4 deletions README.md
Expand Up @@ -49,6 +49,8 @@ See [MAINTAINERS.md](https://github.com/thanos-io/thanos/blob/main/MAINTAINERS.m
The core this module is the [`Bucket` interface](objstore.go):

```go mdox-exec="sed -n '37,50p' objstore.go"
// Bucket provides read and write access to an object storage bucket.
// NOTE: We assume strong consistency for write-read flow.
type Bucket interface {
io.Closer
BucketReader
Expand All @@ -61,13 +63,13 @@ type Bucket interface {
// If object does not exists in the moment of deletion, Delete should throw error.
Delete(ctx context.Context, name string) error

// Name returns the bucket name for the provider.
Name() string
```
All [provider implementations](providers) have to implement `Bucket` interface that allows common read and write operations that all supported by all object providers. If you want to limit the code that will do bucket operation to only read access (smart idea, allowing to limit access permissions), you can use the [`BucketReader` interface](objstore.go):
```go mdox-exec="sed -n '68,88p' objstore.go"

// BucketReader provides read access to an object storage bucket.
type BucketReader interface {
// Iter calls f for each entry in the given directory (not recursive.). The argument to f is the full
// object name including the prefix of the inspected directory.
Expand All @@ -87,8 +89,6 @@ type BucketReader interface {
IsObjNotFoundErr(err error) bool

// Attributes returns information about the specified object.
Attributes(ctx context.Context, name string) (ObjectAttributes, error)
}
```
Those interfaces represent the object storage operations your code can use from `objstore` clients.
Expand Down
6 changes: 3 additions & 3 deletions client/factory.go
Expand Up @@ -12,6 +12,9 @@ import (
"github.com/go-kit/log/level"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
"gopkg.in/yaml.v2"

"github.com/thanos-io/objstore"
"github.com/thanos-io/objstore/providers/azure"
"github.com/thanos-io/objstore/providers/bos"
"github.com/thanos-io/objstore/providers/cos"
Expand All @@ -20,9 +23,6 @@ import (
"github.com/thanos-io/objstore/providers/oss"
"github.com/thanos-io/objstore/providers/s3"
"github.com/thanos-io/objstore/providers/swift"
yaml "gopkg.in/yaml.v2"

"github.com/thanos-io/objstore"
)

type ObjProvider string
Expand Down
65 changes: 65 additions & 0 deletions clientutil/parse.go
@@ -0,0 +1,65 @@
// Copyright (c) The Thanos Authors.
// Licensed under the Apache License 2.0.

package clientutil

import (
"net/http"
"strconv"
"time"

"github.com/pkg/errors"
)

// ParseContentLength returns the content length (in bytes) parsed from the Content-Length
// HTTP header in input.
func ParseContentLength(m http.Header) (int64, error) {
const name = "Content-Length"

v, ok := m[name]
if !ok {
return 0, errors.Errorf("%s header not found", name)
}

if len(v) == 0 {
return 0, errors.Errorf("%s header has no values", name)
}

ret, err := strconv.ParseInt(v[0], 10, 64)
if err != nil {
return 0, errors.Wrapf(err, "convert %s", name)
}

return ret, nil
}

// ParseLastModified returns the timestamp parsed from the Last-Modified
// HTTP header in input.
// Passing an second parameter, named f, to specify the time format.
// If f is empty then RFC3339 will be used as default format.
func ParseLastModified(m http.Header, f string) (time.Time, error) {
const (
name = "Last-Modified"
defaultFormat = time.RFC3339
)

v, ok := m[name]
if !ok {
return time.Time{}, errors.Errorf("%s header not found", name)
}

if len(v) == 0 {
return time.Time{}, errors.Errorf("%s header has no values", name)
}

if f == "" {
f = defaultFormat
}

mod, err := time.Parse(f, v[0])
if err != nil {
return time.Time{}, errors.Wrapf(err, "parse %s", name)
}

return mod, nil
}
110 changes: 110 additions & 0 deletions clientutil/parse_test.go
@@ -0,0 +1,110 @@
// Copyright (c) The Thanos Authors.
// Licensed under the Apache License 2.0.

package clientutil

import (
"net/http"
"testing"
"time"

alioss "github.com/aliyun/aliyun-oss-go-sdk/oss"
"github.com/efficientgo/tools/core/pkg/testutil"
)

func TestParseLastModified(t *testing.T) {
location, _ := time.LoadLocation("GMT")
tests := map[string]struct {
headerValue string
expectedVal time.Time
expectedErr string
format string
}{
"no header": {
expectedErr: "Last-Modified header not found",
},
"empty format string to default RFC3339 format": {
headerValue: "2015-11-06T10:07:11.000Z",
expectedVal: time.Date(2015, time.November, 6, 10, 7, 11, 0, time.UTC),
format: "",
},
"valid RFC3339 header value": {
headerValue: "2015-11-06T10:07:11.000Z",
expectedVal: time.Date(2015, time.November, 6, 10, 7, 11, 0, time.UTC),
format: time.RFC3339,
},
"invalid RFC3339 header value": {
headerValue: "invalid",
expectedErr: `parse Last-Modified: parsing time "invalid" as "2006-01-02T15:04:05Z07:00": cannot parse "invalid" as "2006"`,
format: time.RFC3339,
},
"valid RFC1123 header value": {
headerValue: "Fri, 24 Feb 2012 06:07:48 GMT",
expectedVal: time.Date(2012, time.February, 24, 6, 7, 48, 0, location),
format: time.RFC1123,
},
"invalid RFC1123 header value": {
headerValue: "invalid",
expectedErr: `parse Last-Modified: parsing time "invalid" as "Mon, 02 Jan 2006 15:04:05 MST": cannot parse "invalid" as "Mon"`,
format: time.RFC1123,
},
}

for testName, testData := range tests {
t.Run(testName, func(t *testing.T) {
meta := http.Header{}
if testData.headerValue != "" {
meta.Add(alioss.HTTPHeaderLastModified, testData.headerValue)
}

actual, err := ParseLastModified(meta, testData.format)

if testData.expectedErr != "" {
testutil.NotOk(t, err)
testutil.Equals(t, testData.expectedErr, err.Error())
} else {
testutil.Ok(t, err)
testutil.Assert(t, testData.expectedVal.Equal(actual))
}
})
}
}

func TestParseContentLength(t *testing.T) {
tests := map[string]struct {
headerValue string
expectedVal int64
expectedErr string
}{
"no header": {
expectedErr: "Content-Length header not found",
},
"invalid header value": {
headerValue: "invalid",
expectedErr: `convert Content-Length: strconv.ParseInt: parsing "invalid": invalid syntax`,
},
"valid header value": {
headerValue: "12345",
expectedVal: 12345,
},
}

for testName, testData := range tests {
t.Run(testName, func(t *testing.T) {
meta := http.Header{}
if testData.headerValue != "" {
meta.Add(alioss.HTTPHeaderContentLength, testData.headerValue)
}

actual, err := ParseContentLength(meta)

if testData.expectedErr != "" {
testutil.NotOk(t, err)
testutil.Equals(t, testData.expectedErr, err.Error())
} else {
testutil.Ok(t, err)
testutil.Equals(t, testData.expectedVal, actual)
}
})
}
}
1 change: 1 addition & 0 deletions go.mod
Expand Up @@ -26,6 +26,7 @@ require (
github.com/tencentyun/cos-go-sdk-v5 v0.7.34
go.uber.org/atomic v1.9.0
golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5
golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f
google.golang.org/api v0.80.0
gopkg.in/alecthomas/kingpin.v2 v2.2.6
gopkg.in/yaml.v2 v2.4.0
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Expand Up @@ -548,6 +548,8 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f h1:Ax0t5p6N38Ga0dThY21weqDEyz2oklo4IvDkpigvkD8=
golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
Expand Down

0 comments on commit ce33880

Please sign in to comment.