feat: pull over HTTP, storage cache

Author: Spelkington

Flowthru.slnx

@@ -16,6 +16,7 @@
     <Project Path="src/extensions/Flowthru.Extensions.Excel/Flowthru.Extensions.Excel.csproj" />
     <Project Path="src/extensions/Flowthru.Extensions.EFCore/Flowthru.Extensions.EFCore.csproj" />
     <Project Path="src/extensions/Flowthru.Extensions.GQL/Flowthru.Extensions.GQL.csproj" />
+    <Project Path="src/extensions/Flowthru.Extensions.Http/Flowthru.Extensions.Http.csproj" />
     <Project Path="src/extensions/Flowthru.Extensions.MLNet/Flowthru.Extensions.MLNet.csproj" />
     <Project Path="src/extensions/Flowthru.Extensions.Parquet/Flowthru.Extensions.Parquet.csproj" />
     <Project Path="src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.csproj" />

docs/reference/misc/external/databricks-appkit/.source.env

@@ -0,0 +1,2 @@
+SOURCE_TYPE=repo
+SOURCE_ADDRESS=https://github.com/databricks/appkit.git

docs/reference/misc/external/databricks-cli/.source.env

@@ -0,0 +1,2 @@
+SOURCE_TYPE=repo
+SOURCE_ADDRESS=https://github.com/databricks/cli.git

examples/advanced/RetailDataSplitFlow/.gitignore

@@ -1,6 +1,9 @@
 # Generated metadata
 Metadata/
 
+# HTTP response cache (populated at runtime by CachedHttpStorageMedium)
+.http-cache/
+
 # Ignore all generated data artifacts
 Data/**
 
@@ -9,8 +12,9 @@ Data/**
 !Data/*/Schemas/
 !Data/_01_Raw/Datasets/
 
-# Allow raw input dataset files
-!Data/_01_Raw/Datasets/*
+# Allow fixture files in the raw Datasets directory (JSON lookups, etc.)
+# Raw CSV data is fetched at runtime via HttpStorageMedium — do not commit large CSVs.
+!Data/_01_Raw/Datasets/*.json
 
 # Allow all schema and catalog .cs files
 !Data/**/*.cs

examples/advanced/RetailDataSplitFlow/Data/CoreCatalog.cs

@@ -1,14 +1,17 @@
 using Flowthru.Core.Data;
+using Flowthru.Core.Data.Storage;
 
 namespace RetailDataMultipipeline.Data;
 
 public partial class CoreCatalog : CatalogAbstract
 {
   private readonly string _basePath;
+  private readonly IStorageMediumResolver? _resolver;
 
-  public CoreCatalog(string basePath)
+  public CoreCatalog(string basePath, IStorageMediumResolver? resolver = null)
   {
     _basePath = basePath;
+    _resolver = resolver;
     InitializeCatalogProperties();
   }
 }

examples/advanced/RetailDataSplitFlow/Data/_01_Raw/CoreCatalog.Raw.cs

@@ -6,15 +6,18 @@ namespace RetailDataMultipipeline.Data;
 public partial class CoreCatalog
 {
   /// <summary>
-  /// All daily retail transaction CSV files from the by-day directory, read as a
-  /// single concatenated sequence. Read-only — immutable raw source data.
+  /// Full online-retail dataset downloaded from the Spark: The Definitive Guide
+  /// GitHub repository. The resolver routes this https:// URI through
+  /// HttpStorageMedium at runtime; local file paths fall back to FileStorageMedium.
+  /// Read-only — immutable raw source data.
   /// </summary>
   public IItem<IEnumerable<RetailTransactionSchema>> RetailTransactionsRaw =>
     CreateItem(
       () =>
-        ItemFactory.Enumerable.CsvDirectory<RetailTransactionSchema>(
+        ItemFactory.Enumerable.Csv<RetailTransactionSchema>(
           label: "RetailTransactionsRaw",
-          directoryPath: $"{_basePath}/_01_Raw/Datasets"
+          filePath: "https://raw.githubusercontent.com/databricks/Spark-The-Definitive-Guide/refs/heads/master/data/retail-data/all/online-retail-dataset.csv",
+          resolver: _resolver
         )
     );