feat: refactor cache handling, move storage out of generated

.gitignore

@@ -61,3 +61,6 @@ report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
 
 # Output of the go coverage tool, specifically when used with LiteIDE
 *.out
+
+# WunderGraph
+**/.wundergraph/cache/

.prettierignore

@@ -5,7 +5,8 @@ pnpm-lock.yaml
 *lock.json
 
 # WunderGraph
-generated
+**/.wundergraph/cache
+**/.wundergraph/generated
 
 # Next.js build output
 .next

cli/commands/generate.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"os"
 	"path"
-	"time"
 
 	"github.com/jensneuse/abstractlogger"
 	"github.com/spf13/cobra"
@@ -19,14 +18,18 @@ import (
 
 var (
 	generateAndPublish bool
+	offline            bool
 )
 
 // generateCmd represents the generate command
 var generateCmd = &cobra.Command{
 	Use:   "generate",
-	Short: "Runs the code generation process once",
-	Long: `In contrast to wunderctl up, it only generates all files in ./generated but doesn't start WunderGraph or the hooks.
-Use this command if you only want to generate the configuration`,
+	Short: "Generate the production config",
+	Long: `Generate the production config to start the node and hook component
+ with 'wunderctl start' or individually with 'wunderctl node start', 'wunderctl
+ server start'. All files are stored to .wundergraph/generated. The local
+ introspection cache has precedence. You can overwrite this behavior by passing
+ --no-cache to the command`,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		wunderGraphDir, err := files.FindWunderGraphDir(_wunderGraphDirConfig)
 		if err != nil {
@@ -41,8 +44,7 @@ Use this command if you only want to generate the configuration`,
 		// optional, no error check
 		codeServerFilePath, _ := files.CodeFilePath(wunderGraphDir, serverEntryPointFilename)
 
-		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
-		defer cancel()
+		ctx := context.Background()
 
 		configOutFile := path.Join("generated", "bundle", "config.js")
 
@@ -57,6 +59,8 @@ Use this command if you only want to generate the configuration`,
 				// Run scripts in prod mode
 				"NODE_ENV=production",
 				fmt.Sprintf("WUNDERGRAPH_PUBLISH_API=%t", generateAndPublish),
+				fmt.Sprintf("WG_ENABLE_INTROSPECTION_CACHE=%t", !disableCache),
+				fmt.Sprintf("WG_ENABLE_INTROSPECTION_OFFLINE=%t", offline),
 				fmt.Sprintf("WG_DIR_ABS=%s", wunderGraphDir),
 			),
 		})
@@ -172,5 +176,6 @@ Use this command if you only want to generate the configuration`,
 
 func init() {
 	generateCmd.Flags().BoolVarP(&generateAndPublish, "publish", "p", false, "publish the generated API immediately")
+	generateCmd.Flags().BoolVar(&offline, "offline", false, "disables loading resources from the network")
 	rootCmd.AddCommand(generateCmd)
 }

cli/commands/root.go

@@ -1,10 +1,12 @@
 package commands
 
 import (
+	"errors"
 	"fmt"
 	"io/fs"
 	"net/http"
 	"os"
+	"path/filepath"
 	"time"
 
 	"github.com/fatih/color"
@@ -38,6 +40,8 @@ var (
 	jsonEncodedLogging    bool
 	serviceToken          string
 	_wunderGraphDirConfig string
+	disableCache          bool
+	clearCache            bool
 
 	red    = color.New(color.FgHiRed)
 	green  = color.New(color.FgHiGreen)
@@ -70,16 +74,27 @@ You can opt out of this by setting the following environment variable: WUNDERGRA
 		if err != nil {
 			if _, ok := err.(*fs.PathError); ok {
 				log.Debug("starting without env file")
-				return nil
+			} else {
+				log.Fatal("error loading env file",
+					abstractlogger.Error(err))
 			}
-			log.Fatal("error loading env file",
-				abstractlogger.Error(err),
-			)
 		} else {
 			log.Debug("env file successfully loaded",
 				abstractlogger.String("file", DotEnvFile),
 			)
 		}
+
+		if clearCache {
+			wunderGraphDir, err := files.FindWunderGraphDir(_wunderGraphDirConfig)
+			if err != nil {
+				return err
+			}
+			cacheDir := filepath.Join(wunderGraphDir, "cache")
+			if err := os.RemoveAll(cacheDir); err != nil && !errors.Is(err, os.ErrNotExist) {
+				return err
+			}
+		}
+
 		return nil
 	},
 	RunE: func(cmd *cobra.Command, args []string) error {
@@ -144,6 +159,8 @@ func init() {
 	rootCmd.PersistentFlags().BoolVar(&enableDebugMode, "debug", false, "enables the debug mode so that all requests and responses will be logged")
 	rootCmd.PersistentFlags().BoolVar(&jsonEncodedLogging, "json-encoded-logging", false, "switches the logging to json encoded logging")
 	rootCmd.PersistentFlags().StringVar(&_wunderGraphDirConfig, "wundergraph-dir", files.WunderGraphDirName, "path to your .wundergraph directory")
+	rootCmd.PersistentFlags().BoolVar(&disableCache, "no-cache", false, "disables local caches")
+	rootCmd.PersistentFlags().BoolVar(&clearCache, "clear-cache", false, "clears local caches during startup")
 }
 
 func buildLogger(level abstractlogger.Level) abstractlogger.Logger {

cli/commands/up.go

@@ -21,10 +21,6 @@ import (
 	"github.com/wundergraph/wundergraph/pkg/webhooks"
 )
 
-var (
-	clearIntrospectionCache bool
-)
-
 // upCmd represents the up command
 var upCmd = &cobra.Command{
 	Use:   "up",
@@ -62,20 +58,7 @@ var upCmd = &cobra.Command{
 			abstractlogger.String("builtBy", BuildInfo.BuiltBy),
 		)
 
-		introspectionCacheDir := path.Join(wunderGraphDir, "generated", "introspection", "cache")
-		_, errIntrospectionDir := os.Stat(introspectionCacheDir)
-		if errIntrospectionDir == nil {
-			if clearIntrospectionCache {
-				err = os.RemoveAll(introspectionCacheDir)
-				if err != nil {
-					return err
-				}
-			}
-		}
-		err = os.MkdirAll(introspectionCacheDir, os.ModePerm)
-		if err != nil {
-			return err
-		}
+		introspectionCacheDir := path.Join(wunderGraphDir, "cache", "introspection")
 
 		configJsonPath := path.Join(wunderGraphDir, "generated", configJsonFilename)
 		webhooksDir := path.Join(wunderGraphDir, webhooks.WebhookDirectoryName)
@@ -93,18 +76,8 @@ var upCmd = &cobra.Command{
 			AbsWorkingDir: wunderGraphDir,
 			ScriptArgs:    []string{configOutFile},
 			Logger:        log,
-			FirstRunEnv: append(os.Environ(),
-				// when the user runs `wunderctl up` for the first time, we revalidate the cache
-				// so the user can be sure that the introspection is up-to-date. In case of an API is not available
-				// we will fall back to the cached introspection (when available)
-				"WG_ENABLE_INTROSPECTION_CACHE=false",
-				// this option allows us to make different decision for the first run
-				// for example, we decide to not use the cache, but we will prefill the cache
-				"WG_DEV_FIRST_RUN=true",
-				fmt.Sprintf("WG_DIR_ABS=%s", wunderGraphDir),
-			),
 			ScriptEnv: append(os.Environ(),
-				"WG_ENABLE_INTROSPECTION_CACHE=true",
+				fmt.Sprintf("WG_ENABLE_INTROSPECTION_CACHE=%t", !disableCache),
 				fmt.Sprintf("WG_DIR_ABS=%s", wunderGraphDir),
 			),
 		})
@@ -119,6 +92,7 @@ var upCmd = &cobra.Command{
 			ScriptEnv: append(os.Environ(),
 				// this environment variable starts the config runner in "Polling Mode"
 				"WG_DATA_SOURCE_POLLING_MODE=true",
+				fmt.Sprintf("WG_ENABLE_INTROSPECTION_CACHE=%t", !disableCache),
 				fmt.Sprintf("WG_DIR_ABS=%s", wunderGraphDir),
 			),
 		})
@@ -300,5 +274,4 @@ var upCmd = &cobra.Command{
 
 func init() {
 	rootCmd.AddCommand(upCmd)
-	upCmd.Flags().BoolVar(&clearIntrospectionCache, "clear-introspection-cache", false, "clears the introspection cache")
 }

docs-website/src/pages/docs/wundergraph-config-ts-reference/configure-introspection.md

@@ -25,15 +25,32 @@ That's why we've introduced introspection caching.
 
 {% callout type="warning" %}
 By default, every data source is introspected exactly once.
-The result is cached in the `.wundergraph/generated/introspection` directory.
+The result is cached in the `.wundergraph/cache/introspection` directory.
 {% /callout %}
 
-If a cache entry is found for a data source,
-it's always used instead of introspecting the data source again.
+When caching is enabled, if a cache entry is found for a data source,
+it's always used instead of introspecting the data source again. Cache is
+optional and might be turned off globally using the `--no-cache` flag.
 
 ## Clearing the Cache
 
-It's possible to clear the cache by running `wunderctl up --clear-introspection-cache`.
+Cache can be emptied when `wunderctl` starts by using the `--clear-cache`
+flag. Additionally, `wunderctl up` will automatically clear the cache
+when it starts up. To clear the cache again, just restart it.
+
+## Pregenerating the Cache
+
+For CI environments that cannot perform network requests, it might be
+useful to generate the cache contents and make the tests run offline.
+
+To populate the cache contents run `wunderctl generate --clear-cache`.
+This will delete any previous cache, introspect the data sources, and
+finally write the cache entries. `.wunderctl/cache` can then be stored
+and deployed to CI runners.
+
+To verify that your tests can run without fetching data from the network
+in a connected machine, the `--offline` flag can be used to disable loading
+remote resources.
 
 ## Enable Introspection Polling
 
@@ -56,7 +73,7 @@ const spaceX = introspect.graphql({
 
 ## Disable Introspection Caching
 
-It's also possible to disable introspection caching.
+It's also possible to disable introspection caching for a given data source.
 Set `disableCache` to true, and `wunderctl up` will ignore the cache for this data source.
 
 ```typescript

examples/apollo-federation/.gitignore

@@ -3,14 +3,15 @@
 .vscode
 .idea
 
+# WunderGraph build output
+.wundergraph/cache
+.wundergraph/generated
+
 # dependencies
 node_modules
 /.pnp
 .pnp.js
 
-# WunderGraph build output
-.wundergraph/generated
-
 # testing
 /coverage
 

examples/auth0-oidc-authentication/.gitignore

@@ -3,14 +3,15 @@
 .vscode
 .idea
 
+# WunderGraph build output
+.wundergraph/cache
+.wundergraph/generated
+
 # dependencies
 node_modules
 /.pnp
 .pnp.js
 
-# WunderGraph build output
-.wundergraph/generated
-
 # testing
 /coverage
 

examples/caching/.gitignore

@@ -3,14 +3,15 @@
 .vscode
 .idea
 
+# WunderGraph build output
+.wundergraph/cache
+.wundergraph/generated
+
 # dependencies
 node_modules
 /.pnp
 .pnp.js
 
-# WunderGraph build output
-.wundergraph/generated
-
 # testing
 /coverage
 

examples/cross-api-joins/.gitignore

@@ -4,6 +4,7 @@
 .idea
 
 # WunderGraph build output
+.wundergraph/cache
 .wundergraph/generated
 
 # dependencies

examples/faunadb-nextjs/.gitignore

@@ -3,14 +3,15 @@
 .vscode
 .idea
 
+# WunderGraph build output
+.wundergraph/cache
+.wundergraph/generated
+
 # dependencies
 node_modules
 /.pnp
 .pnp.js
 
-# WunderGraph build output
-.wundergraph/generated
-
 # testing
 /coverage
 

examples/fragments/.gitignore

@@ -4,6 +4,7 @@
 .idea
 
 # WunderGraph build output
+.wundergraph/cache
 .wundergraph/generated
 
 # dependencies

examples/golang-client/.gitignore

@@ -4,6 +4,7 @@
 .idea
 
 # WunderGraph build output
+.wundergraph/cache
 .wundergraph/generated
 
 # dependencies

examples/graphql-sse-subscriptions/.gitignore

@@ -4,6 +4,7 @@
 .idea
 
 # WunderGraph build output
+.wundergraph/cache
 .wundergraph/generated
 
 # dependencies

examples/graphql-yoga-sse-subscriptions/.gitignore

@@ -4,6 +4,7 @@
 .idea
 
 # WunderGraph build output
+.wundergraph/cache
 .wundergraph/generated
 
 # dependencies

examples/hooks/.gitignore

@@ -3,14 +3,15 @@
 .vscode
 .idea
 
+# WunderGraph build output
+.wundergraph/cache
+.wundergraph/generated
+
 # dependencies
 node_modules
 /.pnp
 .pnp.js
 
-# WunderGraph build output
-.wundergraph/generated
-
 # testing
 /coverage