feat: gql query object for better control over materializing data

Author: Spelkington

.github/workflows/release.yml

@@ -7,7 +7,17 @@ on:
       - "**.md"
       - "docs/**"
       - ".github/instructions/**"
-  workflow_dispatch: ~
+  workflow_dispatch:
+    inputs:
+      force_bump:
+        description: "Force a version bump type (bypasses conventional commits). Defaults to 'minor' on manual dispatch."
+        required: false
+        default: minor
+        type: choice
+        options:
+          - minor
+          - patch
+          - major
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -127,7 +137,8 @@ jobs:
           CI: "true"
         run: |
           OLD_VERSION=$(node -p "require('./package.json').version")
-          node scripts/release.mjs --from="${{ steps.last-published.outputs.tag }}"
+          FORCE_BUMP_FLAG=${{ inputs.force_bump && format('--force-bump={0}', inputs.force_bump) || '' }}
+          node scripts/release.mjs --from="${{ steps.last-published.outputs.tag }}" ${FORCE_BUMP_FLAG}
           NEW_VERSION=$(node -p "require('./package.json').version")
           echo "version=$NEW_VERSION" >> "$GITHUB_OUTPUT"
           if [ "$OLD_VERSION" != "$NEW_VERSION" ]; then

CONTRIBUTING.md

@@ -105,7 +105,7 @@ The project uses NX for task orchestration. Common commands:
 
 ```bash
 dotnet build # Confirm solution builds fully
-nx run affected -t test # Run all test projects affected by current changes
+nx run affected -t test # IMPORTANT: Run all test projects affected by current changes
 dotnet test # Run all tests for the project.
 ```
 

examples/advanced/KedroSpaceflightsGQL/Data/_01_Raw/Catalog.Raw.cs

@@ -54,14 +54,20 @@ public partial class Catalog
     CreateItem(() => ItemFactory.Single.Memory<bool>("GqlDatabaseSeeded"));
 
   // ── Raw GQL entries — consumed by the DataProcessing flow ───────────────
+  //
+  // These are DEFERRED query handles, not materialized collections. No network I/O happens
+  // when the catalog is constructed or during pre-flight (beyond a lightweight connectivity
+  // probe). The step that consumes these entries calls .ToList() to trigger the actual
+  // GQL fetch — see CreateModelInputTableStep for the materialization point.
 
   /// <summary>
-  /// Raw company data queried from the GQL server.
+  /// Deferred GQL query handle for company data. The consuming step materializes this
+  /// via <c>ToList()</c>, which fires the <c>GetCompanies</c> query against the server.
   /// </summary>
-  public IItem<IEnumerable<IGetCompanies_Companies>> Companies =>
+  public IItem<GqlQuery<IGetCompaniesResult, IGetCompanies_Companies>> Companies =>
     CreateItem(
       () =>
-        GqlItemFactory.Enumerable.Query<IGetCompaniesResult, IGetCompanies_Companies>(
+        GqlItemFactory.Query.NonPaged<IGetCompaniesResult, IGetCompanies_Companies>(
           label: "GQLCompanies",
           queryFunc: ct => _client.GetCompanies.ExecuteAsync(ct),
           selectData: r => r.Companies,
@@ -70,12 +76,13 @@ public partial class Catalog
     );
 
   /// <summary>
-  /// Raw shuttle data queried from the GQL server.
+  /// Deferred GQL query handle for shuttle data. The consuming step materializes this
+  /// via <c>ToList()</c>, which fires the <c>GetShuttles</c> query against the server.
   /// </summary>
-  public IItem<IEnumerable<IGetShuttles_Shuttles>> Shuttles =>
+  public IItem<GqlQuery<IGetShuttlesResult, IGetShuttles_Shuttles>> Shuttles =>
     CreateItem(
       () =>
-        GqlItemFactory.Enumerable.Query<IGetShuttlesResult, IGetShuttles_Shuttles>(
+        GqlItemFactory.Query.NonPaged<IGetShuttlesResult, IGetShuttles_Shuttles>(
           label: "GQLShuttles",
           queryFunc: ct => _client.GetShuttles.ExecuteAsync(ct),
           selectData: r => r.Shuttles,
@@ -84,12 +91,13 @@ public partial class Catalog
     );
 
   /// <summary>
-  /// Raw review data queried from the GQL server.
+  /// Deferred GQL query handle for review data. The consuming step materializes this
+  /// via <c>ToList()</c>, which fires the <c>GetReviews</c> query against the server.
   /// </summary>
-  public IItem<IEnumerable<IGetReviews_Reviews>> Reviews =>
+  public IItem<GqlQuery<IGetReviewsResult, IGetReviews_Reviews>> Reviews =>
     CreateItem(
       () =>
-        GqlItemFactory.Enumerable.Query<IGetReviewsResult, IGetReviews_Reviews>(
+        GqlItemFactory.Query.NonPaged<IGetReviewsResult, IGetReviews_Reviews>(
           label: "GQLReviews",
           queryFunc: ct => _client.GetReviews.ExecuteAsync(ct),
           selectData: r => r.Reviews,

examples/advanced/KedroSpaceflightsGQL/Flows/DataProcessing/Steps/CreateModelInputTableStep.cs

@@ -1,31 +1,54 @@
 using Flowthru.Core.Steps;
+using Flowthru.Extensions.GQL.Data;
 using KedroSpaceflightsGQL.Data._03_Primary.Schemas;
 using KedroSpaceflightsGQL.Infra.GqlClient;
 
 namespace KedroSpaceflightsGQL.Flows.DataProcessing.Steps;
 
 /// <summary>
-/// Joins typed shuttle, company, and review data from the GQL server into a unified model
+/// Materializes deferred GQL query handles and joins the results into a unified model
 /// input table. All fields are already strongly-typed — no parsing required.
-/// The <c>bool</c> first input is the GqlDatabaseSeeded gate; it is consumed only to
-/// express the DAG dependency on Ingest and is otherwise unused.
 /// </summary>
+/// <remarks>
+/// <para>
+/// The <c>bool</c> first input is the <c>GqlDatabaseSeeded</c> gate; it is consumed only
+/// to express the DAG dependency on Ingest and is otherwise unused.
+/// </para>
+/// <para>
+/// The remaining three inputs are <see cref="GqlQuery{TResult,T}"/> handles — deferred
+/// query descriptors that carry the connection details and pagination config but have not
+/// yet executed any network calls. The calls to <c>ToList()</c> below are the
+/// materialization points: each one fires the corresponding GQL query (paginating as
+/// needed) and pulls the full dataset into memory before the join.
+/// </para>
+/// <para>
+/// This is the step-level analog of <c>TypedFrame&lt;T&gt;.ToList()</c> in the Spark
+/// extension: the catalog declares <em>what</em> to query; the step decides <em>when</em>
+/// to materialize and <em>how</em> to combine the results.
+/// </para>
+/// </remarks>
 [FlowthruStep]
 public static class CreateModelInputTableStep
 {
   public static Func<
     (
       bool,
-      IEnumerable<IGetShuttles_Shuttles>,
-      IEnumerable<IGetCompanies_Companies>,
-      IEnumerable<IGetReviews_Reviews>
+      GqlQuery<IGetShuttlesResult, IGetShuttles_Shuttles>,
+      GqlQuery<IGetCompaniesResult, IGetCompanies_Companies>,
+      GqlQuery<IGetReviewsResult, IGetReviews_Reviews>
     ),
     IEnumerable<ModelInputTableSchema>
   > Create()
   {
     return (input) =>
     {
-      var (_, shuttles, companies, reviews) = input;
+      var (_, shuttlesQuery, companiesQuery, reviewsQuery) = input;
+
+      // Materialization — each call fires a GQL query (with pagination if configured).
+      // Network I/O happens here, not in the catalog.
+      var shuttles = shuttlesQuery.ToList();
+      var companies = companiesQuery.ToList();
+      var reviews = reviewsQuery.ToList();
 
       // Join reviews to shuttles
       var ratedShuttles = reviews
@@ -38,7 +61,7 @@ public static Func<
         .ToList();
 
       // Join with companies
-      var modelInputTable = ratedShuttles
+      return ratedShuttles
         .Join(
           companies,
           rs => rs.Shuttle.CompanyId,
@@ -61,8 +84,6 @@ public static Func<
             }
         )
         .ToList();
-
-      return modelInputTable;
     };
   }
 }

scripts/release.mjs

@@ -26,12 +26,55 @@ const ROOT = resolve(__dirname, '..');
 const DRY_RUN = process.argv.includes('--dry-run');
 const FROM_ARG = process.argv.find(a => a.startsWith('--from='));
 const FROM_TAG = FROM_ARG ? FROM_ARG.slice('--from='.length) : undefined;
+const FORCE_BUMP_ARG = process.argv.find(a => a.startsWith('--force-bump='));
+const FORCE_BUMP = FORCE_BUMP_ARG ? FORCE_BUMP_ARG.slice('--force-bump='.length) : undefined;
 
 // ── 1. Version determination via NX Release ──────────────────────────────────
+//
+// NX's conventional-commits specifier detection filters commits by project-file
+// ownership — only commits that touch files belonging to a project in the release
+// group are considered. Since the release group only contains the workspace root
+// project (`flowthru`), commits that exclusively touch src/ library projects are
+// silently discarded by NX before it can compute a specifier.
+//
+// To compensate, we pre-scan the git log ourselves and derive the specifier from
+// commit types, then hand it to NX as an explicit override. NX still handles
+// version writing, changelog generation, and tagging — we're only fixing the
+// commit-detection blind spot.
+//
+// Commit types and their bumps mirror nx.json release.conventionalCommits.types:
+//   feat         → minor
+//   fix/perf/revert → patch
+//   breaking (! or BREAKING CHANGE) → major
+//   everything else → not releasable
+function deriveSpecifierFromGitLog(fromRef) {
+  const range = fromRef ? `${fromRef}..HEAD` : 'HEAD';
+  const raw = execSync(`git log ${range} --format=%s`).toString().trim();
+  if (!raw) return null;
+  const lines = raw.split('\n').filter(Boolean);
+  let hasFeat = false;
+  let hasPatch = false;
+  for (const msg of lines) {
+    if (/^[a-z]+(\([^)]+\))?!:/.test(msg) || /^BREAKING CHANGE/.test(msg)) return 'major';
+    if (/^feat(\([^)]+\))?:/.test(msg)) hasFeat = true;
+    if (/^(fix|perf|revert)(\([^)]+\))?:/.test(msg)) hasPatch = true;
+  }
+  if (hasFeat) return 'minor';
+  if (hasPatch) return 'patch';
+  return null;
+}
+
+const specifier = FORCE_BUMP ?? deriveSpecifierFromGitLog(FROM_TAG);
+
+if (!specifier) {
+  console.log('No version bump required — no releasable commits since last tag.');
+  process.exit(0);
+}
 
 const { workspaceVersion, projectsVersionData } = await releaseVersion({
   dryRun: DRY_RUN,
   verbose: false,
+  specifier,
   ...(FROM_TAG ? { from: FROM_TAG } : {}),
 });