feat: Use stale cache as fallback temporarily

internal/pkg/service/appproxy/appconfig/appconfig.go

@@ -2,14 +2,20 @@ package appconfig
 
 import (
 	"context"
+	"net/http"
 	"time"
 
 	"github.com/benbjohnson/clock"
 	"github.com/keboola/go-client/pkg/client"
 	"github.com/keboola/go-client/pkg/request"
+	"go.opentelemetry.io/otel/attribute"
+
+	"github.com/keboola/keboola-as-code/internal/pkg/log"
+	"github.com/keboola/keboola-as-code/internal/pkg/utils/errors"
 )
 
 type Loader struct {
+	logger log.Logger
 	clock  clock.Clock
 	sender request.Sender
 	cache  map[string]cacheItem
@@ -21,14 +27,17 @@ type cacheItem struct {
 	expiresAt time.Time
 }
 
-func NewLoader(clock clock.Clock, baseURL string) *Loader {
+func NewLoader(logger log.Logger, clock clock.Clock, baseURL string) *Loader {
 	return &Loader{
+		logger: logger,
 		clock:  clock,
 		sender: client.New().WithBaseURL(baseURL),
 		cache:  make(map[string]cacheItem),
 	}
 }
 
+const staleCacheFallbackDuration = time.Hour
+
 func (l *Loader) LoadConfig(ctx context.Context, appID string) (AppProxyConfig, error) {
 	var config *AppProxyConfig
 	var err error
@@ -43,7 +52,7 @@ func (l *Loader) LoadConfig(ctx context.Context, appID string) (AppProxyConfig,
 		// API request with cached ETag
 		config, err = GetAppProxyConfig(l.sender, appID, item.eTag).Send(ctx)
 		if err != nil {
-			return AppProxyConfig{}, err
+			return l.handleError(ctx, appID, now, err, &item)
 		}
 
 		// Update expiration and use the cached config if ETag is still the same
@@ -59,7 +68,7 @@ func (l *Loader) LoadConfig(ctx context.Context, appID string) (AppProxyConfig,
 		// API request without ETag because cache is empty
 		config, err = GetAppProxyConfig(l.sender, appID, "").Send(ctx)
 		if err != nil {
-			return AppProxyConfig{}, err
+			return l.handleError(ctx, appID, now, err, nil)
 		}
 	}
 
@@ -71,3 +80,27 @@ func (l *Loader) LoadConfig(ctx context.Context, appID string) (AppProxyConfig,
 	}
 	return *config, nil
 }
+
+func (l *Loader) handleError(ctx context.Context, appID string, now time.Time, err error, fallbackItem *cacheItem) (AppProxyConfig, error) {
+	var sandboxesError *SandboxesError
+	errors.As(err, &sandboxesError)
+	if sandboxesError == nil && sandboxesError.StatusCode() == http.StatusNotFound {
+		return AppProxyConfig{}, err
+	}
+
+	logger := l.logger
+	if sandboxesError != nil {
+		logger = l.logger.With(attribute.String("exceptionId", sandboxesError.ExceptionID))
+	}
+
+	// An error other than 404 is considered a temporary failure. Keep using the stale cache for staleCacheFallbackDuration as fallback.
+	if fallbackItem != nil && now.Before(fallbackItem.expiresAt.Add(staleCacheFallbackDuration)) {
+		logger.Warnf(ctx, `Using stale cache for app "%s": %s`, appID, err.Error())
+
+		return fallbackItem.config, nil
+	}
+
+	logger.Errorf(ctx, `Failed loading config for app "%s": %s`, appID, err.Error())
+
+	return AppProxyConfig{}, err
+}

internal/pkg/service/appproxy/appconfig/appconfig_test.go

@@ -13,6 +13,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
+	"github.com/keboola/keboola-as-code/internal/pkg/log"
 	"github.com/keboola/keboola-as-code/internal/pkg/utils/errors"
 )
 
@@ -220,6 +221,24 @@ func TestLoader_LoadConfig(t *testing.T) {
 						newResponse(t, 500, map[string]any{}, "", ""),
 						newResponse(t, 500, map[string]any{}, "", ""),
 					},
+					expectedETag: `"etag-value"`,
+					expectedConfig: AppProxyConfig{
+						AppID:           "7",
+						UpstreamAppHost: "app.local",
+						eTag:            `"etag-value"`,
+						maxAge:          60 * time.Second,
+					},
+				},
+				{
+					delay: time.Hour,
+					responses: []*http.Response{
+						newResponse(t, 500, map[string]any{}, "", ""),
+						newResponse(t, 500, map[string]any{}, "", ""),
+						newResponse(t, 500, map[string]any{}, "", ""),
+						newResponse(t, 500, map[string]any{}, "", ""),
+						newResponse(t, 500, map[string]any{}, "", ""),
+						newResponse(t, 500, map[string]any{}, "", ""),
+					},
 					expectedETag:      `"etag-value"`,
 					expectedErrorCode: 500,
 				},
@@ -238,7 +257,7 @@ func TestLoader_LoadConfig(t *testing.T) {
 			transport := httpmock.NewMockTransport()
 
 			url := "https://sandboxes.keboola.com"
-			loader := NewLoader(clk, url)
+			loader := NewLoader(log.NewDebugLogger(), clk, url)
 			loader.sender = loader.sender.(client.Client).WithTransport(transport)
 
 			for _, attempt := range tc.attempts {