core/filemgr: use xxhash instead of sha512 for filenames

pomerium · Nov 1, 2023 · 9ac7f53 · 9ac7f53
1 parent 53573dc
commit 9ac7f53
Show file tree

Hide file tree

Showing 6 changed files with 64 additions and 24 deletions.
diff --git a/config/envoyconfig/clusters_test.go b/config/envoyconfig/clusters_test.go
@@ -23,7 +23,7 @@ import (
 func Test_buildPolicyTransportSocket(t *testing.T) {
 	ctx := context.Background()
 	cacheDir, _ := os.UserCacheDir()
-	customCA := filepath.Join(cacheDir, "pomerium", "envoy", "files", "custom-ca-32484c314b584447463735303142374c31414145374650305a525539554938594d524855353757313942494d473847535231.pem")
+	customCA := filepath.Join(cacheDir, "pomerium", "envoy", "files", "custom-ca-57394a4e5157303436544830.pem")
 
 	b := New("local-grpc", "local-http", "local-metrics", filemgr.NewManager(), nil)
 	rootCABytes, _ := getCombinedCertificateAuthority(&config.Config{Options: &config.Options{}})
@@ -406,10 +406,10 @@ func Test_buildPolicyTransportSocket(t *testing.T) {
 						},
 						"tlsCertificates": [{
 							"certificateChain":{
-								"filename": "`+filepath.Join(cacheDir, "pomerium", "envoy", "files", "tls-crt-354e49305a5a39414a545530374e58454e48334148524c4e324258463837364355564c4e4532464b54355139495547514a38.pem")+`"
+								"filename": "`+filepath.Join(cacheDir, "pomerium", "envoy", "files", "tls-crt-32375a484d4f49594c4d374830.pem")+`"
 							},
 							"privateKey": {
-								"filename": "`+filepath.Join(cacheDir, "pomerium", "envoy", "files", "tls-key-3350415a38414e4e4a4655424e55393430474147324651433949384e485341334b5157364f424b4c5856365a545937383735.pem")+`"
+								"filename": "`+filepath.Join(cacheDir, "pomerium", "envoy", "files", "tls-key-33393156483053584631414836.pem")+`"
 							}
 						}],
 						"validationContext": {

diff --git a/config/envoyconfig/filemgr/filemgr.go b/config/envoyconfig/filemgr/filemgr.go
@@ -3,20 +3,21 @@ package filemgr
 
 import (
 	"context"
-	"fmt"
 	"os"
 	"path/filepath"
+	"sync"
 
 	envoy_config_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
-	"github.com/martinlindhe/base36"
 
 	"github.com/pomerium/pomerium/internal/log"
-	"github.com/pomerium/pomerium/pkg/cryptutil"
 )
 
 // A Manager manages files for envoy.
 type Manager struct {
 	cfg *config
+
+	initOnce sync.Once
+	initErr  error
 }
 
 // NewManager creates a new Manager.
@@ -27,18 +28,23 @@ func NewManager(options ...Option) *Manager {
 	}
 }
 
+func (mgr *Manager) init() {
+	mgr.initOnce.Do(func() {
+		mgr.initErr = os.MkdirAll(mgr.cfg.cacheDir, 0o700)
+	})
+}
+
 // BytesDataSource returns an envoy config data source based on bytes.
 func (mgr *Manager) BytesDataSource(fileName string, data []byte) *envoy_config_core_v3.DataSource {
-	h := base36.EncodeBytes(cryptutil.Hash("filemgr", data))
-	ext := filepath.Ext(fileName)
-	fileName = fmt.Sprintf("%s-%x%s", fileName[:len(fileName)-len(ext)], h, ext)
-
-	if err := os.MkdirAll(mgr.cfg.cacheDir, 0o700); err != nil {
-		log.Error(context.TODO()).Err(err).Msg("filemgr: error creating cache directory, falling back to inline bytes")
+	mgr.init()
+	if mgr.initErr != nil {
+		log.Error(context.Background()).Err(mgr.initErr).Msg("filemgr: error creating cache directory, falling back to inline bytes")
 		return inlineBytes(data)
 	}
 
+	fileName = GetFileNameWithBytesHash(fileName, data)
 	filePath := filepath.Join(mgr.cfg.cacheDir, fileName)
+
 	if _, err := os.Stat(filePath); os.IsNotExist(err) {
 		err = os.WriteFile(filePath, data, 0o600)
 		if err != nil {

diff --git a/config/envoyconfig/filemgr/filemgr_test.go b/config/envoyconfig/filemgr/filemgr_test.go
@@ -17,7 +17,7 @@ func Test(t *testing.T) {
 		ds := mgr.BytesDataSource("test.txt", []byte{1, 2, 3, 4, 5})
 		assert.Equal(t, &envoy_config_core_v3.DataSource{
 			Specifier: &envoy_config_core_v3.DataSource_Filename{
-				Filename: filepath.Join(dir, "test-353354494b53534a5538435652584d594a5759394d43484f38514b34594b4b524b34515339593249344e4238474a5436414b.txt"),
+				Filename: filepath.Join(dir, "test-32354837325a545944534a4537.txt"),
 			},
 		}, ds)
 		mgr.ClearCache()
@@ -32,7 +32,7 @@ func Test(t *testing.T) {
 		ds := mgr.FileDataSource(tmpFilePath)
 		assert.Equal(t, &envoy_config_core_v3.DataSource{
 			Specifier: &envoy_config_core_v3.DataSource_Filename{
-				Filename: filepath.Join(dir, "test-34514f59593332445a5649504230484142544c515057383944383730554833564d32574836354654585954304e424f464336.txt"),
+				Filename: filepath.Join(dir, "test-474136555958463735414951.txt"),
 			},
 		}, ds)
 
@@ -41,7 +41,7 @@ func Test(t *testing.T) {
 		ds = mgr.FileDataSource(tmpFilePath)
 		assert.Equal(t, &envoy_config_core_v3.DataSource{
 			Specifier: &envoy_config_core_v3.DataSource_Filename{
-				Filename: filepath.Join(dir, "test-32564e4457304430393559364b5747373138584f484f5a51334d365758584b47364b555a4c444849513241513457323259.txt"),
+				Filename: filepath.Join(dir, "test-3331324c4a35574d5439444d4c.txt"),
 			},
 		}, ds)
 

diff --git a/config/envoyconfig/filemgr/name.go b/config/envoyconfig/filemgr/name.go
@@ -0,0 +1,18 @@
+package filemgr
+
+import (
+	"fmt"
+	"path/filepath"
+
+	"github.com/cespare/xxhash/v2"
+	"github.com/martinlindhe/base36"
+)
+
+// GetFileNameWithBytesHash constructs a filename using a base filename and a hash of
+// the data. For example: GetFileNameWithBytesHash("example.txt", []byte{...}) ==> "example-abcd1234.txt"
+func GetFileNameWithBytesHash(base string, data []byte) string {
+	h := xxhash.Sum64(data)
+	he := base36.Encode(h)
+	ext := filepath.Ext(base)
+	return fmt.Sprintf("%s-%x%s", base[:len(base)-len(ext)], he, ext)
+}
diff --git a/config/envoyconfig/filemgr/name_test.go b/config/envoyconfig/filemgr/name_test.go
@@ -0,0 +1,19 @@
+package filemgr
+
+import (
+	"crypto/rand"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func BenchmarkGetFileNameWithBytesHash(b *testing.B) {
+	bs := make([]byte, 1024*128)
+	_, err := rand.Read(bs)
+	require.NoError(b, err)
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		GetFileNameWithBytesHash("example.crt", bs)
+	}
+}
diff --git a/config/envoyconfig/listeners_test.go b/config/envoyconfig/listeners_test.go
@@ -41,8 +41,8 @@ func testData(t *testing.T, name string, data interface{}) string {
 
 func Test_buildMetricsHTTPConnectionManagerFilter(t *testing.T) {
 	cacheDir, _ := os.UserCacheDir()
-	certFileName := filepath.Join(cacheDir, "pomerium", "envoy", "files", "tls-crt-354e49305a5a39414a545530374e58454e48334148524c4e324258463837364355564c4e4532464b54355139495547514a38.pem")
-	keyFileName := filepath.Join(cacheDir, "pomerium", "envoy", "files", "tls-key-3350415a38414e4e4a4655424e55393430474147324651433949384e485341334b5157364f424b4c5856365a545937383735.pem")
+	certFileName := filepath.Join(cacheDir, "pomerium", "envoy", "files", "tls-crt-32375a484d4f49594c4d374830.pem")
+	keyFileName := filepath.Join(cacheDir, "pomerium", "envoy", "files", "tls-key-33393156483053584631414836.pem")
 
 	b := New("local-grpc", "local-http", "local-metrics", filemgr.NewManager(), nil)
 	li, err := b.buildMetricsListener(&config.Config{
@@ -74,7 +74,7 @@ func Test_buildDownstreamTLSContext(t *testing.T) {
 	b := New("local-grpc", "local-http", "local-metrics", filemgr.NewManager(), nil)
 
 	cacheDir, _ := os.UserCacheDir()
-	clientCAFileName := filepath.Join(cacheDir, "pomerium", "envoy", "files", "client-ca-3533485838304b593757424e3354425157494c4747433534384f474f3631364d5332554c3332485a483834334d50454c344a.pem")
+	clientCAFileName := filepath.Join(cacheDir, "pomerium", "envoy", "files", "client-ca-313754424855313435355a5348.pem")
 
 	t.Run("no-validation", func(t *testing.T) {
 		downstreamTLSContext, err := b.buildDownstreamTLSContextMulti(context.Background(), &config.Config{Options: &config.Options{}}, nil)
@@ -207,8 +207,7 @@ func Test_buildDownstreamTLSContext(t *testing.T) {
 		}}
 
 		maxVerifyDepth = 10
-		downstreamTLSContext, err :=
-			b.buildDownstreamTLSContextMulti(context.Background(), config, nil)
+		downstreamTLSContext, err := b.buildDownstreamTLSContextMulti(context.Background(), config, nil)
 		require.NoError(t, err)
 		testutil.AssertProtoJSONEqual(t, `{
 			"maxVerifyDepth": 10,
@@ -220,8 +219,7 @@ func Test_buildDownstreamTLSContext(t *testing.T) {
 		}`, downstreamTLSContext.GetCommonTlsContext().GetValidationContext())
 
 		maxVerifyDepth = 0
-		downstreamTLSContext, err =
-			b.buildDownstreamTLSContextMulti(context.Background(), config, nil)
+		downstreamTLSContext, err = b.buildDownstreamTLSContextMulti(context.Background(), config, nil)
 		require.NoError(t, err)
 		testutil.AssertProtoJSONEqual(t, `{
 			"onlyVerifyLeafCertCrl": true,
@@ -243,8 +241,7 @@ func Test_buildDownstreamTLSContext(t *testing.T) {
 				},
 			},
 		}}
-		downstreamTLSContext, err :=
-			b.buildDownstreamTLSContextMulti(context.Background(), config, nil)
+		downstreamTLSContext, err := b.buildDownstreamTLSContextMulti(context.Background(), config, nil)
 		require.NoError(t, err)
 		testutil.AssertProtoJSONEqual(t, `{
 			"maxVerifyDepth": 1,