Skip to content

Commit

Permalink
resources: Add key to reources.GetRemote options map
Browse files Browse the repository at this point in the history
If set, `key` will be used as the only cache key element for the resource.

The default behaviour is to calculate the key based on the URL and all the options.

This means that you can now do:

```
{{ $cacheKey := print $url (now.Format "2006-01-02") }}
{{ $resource := resource.GetRemote $url (dict "key" $cacheKey) }}
```

Fixes #9755
  • Loading branch information
bep committed Apr 13, 2022
1 parent f8c4e16 commit 2dbdf38
Show file tree
Hide file tree
Showing 5 changed files with 70 additions and 1 deletion.
14 changes: 14 additions & 0 deletions common/maps/maps.go
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,20 @@ func ToSliceStringMap(in any) ([]map[string]any, error) {
}
}

// LookupEqualFold finds key in m with case insensitive equality checks.
func LookupEqualFold[T any | string](m map[string]T, key string) (T, bool) {
if v, found := m[key]; found {
return v, true
}
for k, v := range m {
if strings.EqualFold(k, key) {
return v, true
}
}
var s T
return s, false
}

type keyRename struct {
pattern glob.Glob
newKey string
Expand Down
23 changes: 23 additions & 0 deletions common/maps/maps_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -171,3 +171,26 @@ func TestRenameKeys(t *testing.T) {
t.Errorf("Expected\n%#v, got\n%#v\n", expected, m)
}
}

func TestLookupEqualFold(t *testing.T) {
c := qt.New(t)

m1 := map[string]any{
"a": "av",
"B": "bv",
}

v, found := LookupEqualFold(m1, "b")
c.Assert(found, qt.IsTrue)
c.Assert(v, qt.Equals, "bv")

m2 := map[string]string{
"a": "av",
"B": "bv",
}

v, found = LookupEqualFold(m2, "b")
c.Assert(found, qt.IsTrue)
c.Assert(v, qt.Equals, "bv")

}
13 changes: 13 additions & 0 deletions docs/content/en/hugo-pipes/introduction.md
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,19 @@ With `resources.GetRemote`, the first argument is a remote URL:

`resources.Get` and `resources.GetRemote` return `nil` if the resource is not found.

### Caching

By default, Hugo calculates a cache key based on the `URL` and the `options` (e.g. headers) given.


{{< new-in "0.97.0" >}} You can override this by setting a `key` in the options map. This can be used to get more fine grained control over how often a remote resource is fetched, e.g.:


```go-html-template
{{ $cacheKey := print $url (now.Format "2006-01-02") }}
{{ $resource := resource.GetRemote $url (dict "key" $cacheKey) }}
```

### Error Handling

{{< new-in "0.91.0" >}}
Expand Down
10 changes: 9 additions & 1 deletion resources/resource_factories/create/remote.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import (
"strings"

"github.com/gohugoio/hugo/common/hugio"
"github.com/gohugoio/hugo/common/maps"
"github.com/gohugoio/hugo/common/types"
"github.com/gohugoio/hugo/helpers"
"github.com/gohugoio/hugo/media"
Expand Down Expand Up @@ -79,7 +80,7 @@ func (c *Client) FromRemote(uri string, optionsm map[string]any) (resource.Resou
return nil, errors.Wrapf(err, "failed to parse URL for resource %s", uri)
}

resourceID := helpers.HashString(uri, optionsm)
resourceID := calculateResourceID(uri, optionsm)

_, httpResponse, err := c.cacheGetResource.GetOrCreate(resourceID, func() (io.ReadCloser, error) {
options, err := decodeRemoteOptions(optionsm)
Expand Down Expand Up @@ -199,6 +200,13 @@ func (c *Client) validateFromRemoteArgs(uri string, options fromRemoteOptions) e
return nil
}

func calculateResourceID(uri string, optionsm map[string]any) string {
if key, found := maps.LookupEqualFold(optionsm, "key"); found {
return helpers.HashString(key)
}
return helpers.HashString(uri, optionsm)
}

func addDefaultHeaders(req *http.Request, accepts ...string) {
for _, accept := range accepts {
if !hasHeaderValue(req.Header, "Accept", accept) {
Expand Down
11 changes: 11 additions & 0 deletions resources/resource_factories/create/remote_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -83,3 +83,14 @@ func TestDecodeRemoteOptions(t *testing.T) {
}

}

func TestCalculateResourceID(t *testing.T) {
c := qt.New(t)

c.Assert(calculateResourceID("foo", nil), qt.Equals, "5917621528921068675")
c.Assert(calculateResourceID("foo", map[string]any{"bar": "baz"}), qt.Equals, "7294498335241413323")

c.Assert(calculateResourceID("foo", map[string]any{"key": "1234", "bar": "baz"}), qt.Equals, "14904296279238663669")
c.Assert(calculateResourceID("asdf", map[string]any{"key": "1234", "bar": "asdf"}), qt.Equals, "14904296279238663669")
c.Assert(calculateResourceID("asdf", map[string]any{"key": "12345", "bar": "asdf"}), qt.Equals, "12191037851845371770")
}

0 comments on commit 2dbdf38

Please sign in to comment.