[pull] master from clockworklabs:master

.gitattributes

@@ -1 +1,3 @@
 **/module_bindings/** linguist-generated=true eol=lf
+*.verified.cs linguist-generated=true eol=lf
+/crates/bindings-csharp/Runtime/Internal/Autogen/*.cs linguist-generated=true

.github/CODEOWNERS

@@ -0,0 +1,4 @@
+/crates/core/src/db/datastore/traits.rs @cloutiertyler
+/rust-toolchain.toml @jdetter @cloutiertyler
+/.github/CODEOWNERS @cloutiertyler
+LICENSE.txt @cloutiertyler

.github/pull_request_template.md

@@ -17,3 +17,11 @@ but also to its interactions with existing and future code.*
 
 *If you answered more than a 2, explain what is complex about the PR,
 and what other components it interacts with in potentially concerning ways.*
+
+# Testing
+
+*Describe any testing you've done, and any testing you'd like your reviewers to do,
+so that you're confident that all the changes work as expected!*
+
+- [x] *Write a test you've completed here.*
+- [ ] *Write a test you want a reviewer to do here, so they can check it off when they're satisfied.*

.github/workflows/benchmarks.yml

@@ -2,7 +2,7 @@ on:
   push:
     branches:
       - master
-      - jgilles/fix-callgrind
+      - jgilles/fix-callgrind-again
 
   workflow_dispatch:
     inputs:
@@ -170,7 +170,7 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 20 # on a successful run, runs in 8 minutes
     container:
-      image: rust:1.72
+      image: rust:1.78
       options: --privileged
     # filter for a comment containing 'benchmarks please'
     if: ${{ github.event_name != 'issue_comment' || (github.event.issue.pull_request && contains(github.event.comment.body, 'benchmarks please')) }}

.github/workflows/bottest.yml

@@ -0,0 +1,133 @@
+on:
+  push:
+    branches:
+      - jgilles/bottest3
+  workflow_dispatch:
+    inputs:
+      pr_number:
+        description: 'Pull Request Number'
+        required: false
+        default: ''
+  issue_comment:
+    types: [created]
+
+name: Automated bot test
+
+env:
+  GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  GH_REPO: ${{ github.repository }}
+
+jobs:
+  autobottest:
+    name: run bot test (30 bots)
+    runs-on: bots-runner
+    permissions: write-all
+    # filter for a comment containing 'benchmarks please'
+    if: ${{ github.event_name != 'issue_comment' || (github.event.issue.pull_request && contains(github.event.comment.body, 'bots please')) }}
+    env:
+      PR_NUMBER: ${{ github.event.inputs.pr_number || github.event.issue.number || null }}
+    steps:
+      - name: Check membership
+        if: ${{ github.event_name == 'issue_comment' }}
+        env:
+          CONTRIB_ORG: clockworklabs
+          COMMENT_AUTHOR: ${{ github.event.comment.user.login }}
+          ORG_READ_TOKEN: ${{ secrets.ORG_READ_TOKEN }}
+        run: |
+          curl -OL https://github.com/cli/cli/releases/download/v2.37.0/gh_2.37.0_linux_amd64.deb && sudo dpkg -i gh_2.37.0_linux_amd64.deb
+          if [[ $(GH_TOKEN=$ORG_READ_TOKEN gh api --paginate /orgs/{owner}/members --jq 'any(.login == env.COMMENT_AUTHOR)') != true ]]; then
+            gh pr comment $PR_NUMBER -b "Sorry, you don't have permission to run bot tests."
+            exit 1
+          fi
+
+      - name: Checkout sources
+        uses: actions/checkout@v4
+
+      - name: Post initial comment
+        run: |
+          if [[ $PR_NUMBER ]]; then
+            comment_parent=issues/$PR_NUMBER
+            comment_update=issues/comments
+          else
+            comment_parent=commits/$GITHUB_SHA
+            comment_update=comments
+          fi
+          comment_body="Bot test in progress..."
+          comment_id=$(gh api "/repos/{owner}/{repo}/$comment_parent/comments" -f body="$comment_body" --jq .id)
+          echo "COMMENT_UPDATE_URL=/repos/{owner}/{repo}/$comment_update/$comment_id" >>$GITHUB_ENV
+
+      - name: find PR branch
+        if: ${{ env.PR_NUMBER }}
+        run: echo "PR_REF=$(gh pr view $PR_NUMBER --json headRefName --jq .headRefName)" >>"$GITHUB_ENV"
+
+      - name: Checkout sources
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ env.PR_REF || github.ref }}
+          fetch-depth: 1
+
+      - name: Install stable toolchain
+        uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          components: clippy
+          toolchain: stable
+          target: wasm32-unknown-unknown
+          override: true
+
+      - name: Build the bots
+        run: |
+          git apply .github/workflows/bottest_tracing_fix.patch
+          cargo build --release --bin spacetime
+
+      - name: Run bot test script
+        run: |
+          echo "We're on a bots runner, so expecting to find the BitCraftBots repo set up in the expected location"
+          echo "Let's hope the bots runner is up to date!!!"
+          SPACETIME_BIN="$PWD/target/release/spacetime"
+          cd ~/BitCraftBots
+          # using 'exec' allows the process to receive graceful termination signals (note: actions is bad)
+          exec python3 -m scripts.bottest "$SPACETIME_BIN"
+
+      - name: Archive results
+        run: |
+          RESULTS_NAME="pr-$PR_NUMBER-$GITHUB_SHA"
+          mkdir -p bottest-results
+          cp ~/BitCraftBots/results.zip bottest-results/$RESULTS_NAME.zip
+
+      - name: Upload bot test results to DO spaces
+        uses: clockworklabs/s3-upload-action@master
+        with:
+          aws_key_id: ${{ secrets.AWS_KEY_ID }}
+          aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY}}
+          aws_bucket: "spacetimedb-private-bottest-results"
+          source_dir: bottest-results
+          endpoint: https://nyc3.digitaloceanspaces.com
+          destination_dir: bottests
+
+      - name: Post comment
+        run: |
+          RESULTS_NAME="pr-$PR_NUMBER-$GITHUB_SHA"
+          BODY="<details><summary>Bot test results</summary>
+
+          Test results are stored on DO Spaces [here](https://cloud.digitalocean.com/spaces/spacetimedb-private-bottest-results?path=bottests%2F). You'll need to log in
+          (we keep these secret).
+
+          The results of this test are named $RESULTS_NAME.zip.
+
+          </details>"
+
+          gh api "$COMMENT_UPDATE_URL" -X PATCH -f body="$BODY"
+
+      - name: Post failure comment
+        if: ${{ failure() && env.COMMENT_UPDATE_URL }}
+        run: |
+          BODY="Bot test failed. Please check [the workflow run](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details."
+          gh api "$COMMENT_UPDATE_URL" -X PATCH -f body="$BODY"
+
+      - name: Clean up
+        if: always()
+        run: |
+          rm -fr /stdb/*
+          rm -fr bottest-results/
+          rm -fr crates/bench/bottest/

.github/workflows/bottest_tracing_fix.patch

@@ -0,0 +1,13 @@
+diff --git a/Cargo.toml b/Cargo.toml
+index ec9ad430..8bd746d6 100644
+--- a/Cargo.toml
++++ b/Cargo.toml
+@@ -218,7 +218,7 @@ tokio-tungstenite = { version = "0.21", features = ["native-tls"] }
+ tokio-util = { version = "0.7.4", features = ["time"] }
+ toml = "0.8"
+ tower-http = { version = "0.5", features = ["cors"] }
+-tracing = { version = "0.1.37", features = ["release_max_level_off"] }
++tracing = { version = "0.1.37" } #, features = ["release_max_level_off"] }
+ tracing-appender = "0.2.2"
+ tracing-core = "0.1.31"
+ tracing-flame = "0.2.0"

.github/workflows/check-pr-base.yml

@@ -0,0 +1,20 @@
+name: Git tree checks
+
+on:
+  pull_request:
+    types: [opened, edited]
+  merge_group:
+permissions: read-all
+
+jobs:
+  check_base_ref:
+    name: Based on `master`
+    runs-on: ubuntu-latest
+    steps:
+      - id: not_based_on_master
+        if: |
+          github.event_name == 'pull_request' &&
+          github.event.pull_request.base.ref != 'master'
+        run: |
+          echo "This PR is not based on master. Please wait until the base PR merges."
+          exit 1

.github/workflows/ci.yml

@@ -4,6 +4,12 @@ on:
     branches:
       - master
   merge_group:
+  workflow_dispatch:
+    inputs:
+      pr_number:
+        description: 'Pull Request Number'
+        required: false
+        default: ''
 
 name: CI
 
@@ -12,17 +18,26 @@ jobs:
     name: Smoketests
     runs-on: spacetimedb-runner
     steps:
+      - name: Find Git ref
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          PR_NUMBER="${{ github.event.inputs.pr_number || null }}"
+          if test -n "${PR_NUMBER}"; then
+            GIT_REF="$( gh pr view --repo clockworklabs/SpacetimeDB $PR_NUMBER --json headRefName --jq .headRefName )"
+          else
+            GIT_REF="${{ github.ref }}"
+          fi
+          echo "GIT_REF=${GIT_REF}" >>"$GITHUB_ENV"
       - name: Checkout sources
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ env.GIT_REF }}
       - uses: dsherret/rust-toolchain-file@v1
       - name: Start containers
         run: docker compose up -d
       - name: Run smoketests
-        run: test/run-smoke-tests.sh --parallel -x bitcraftmini-pretest zz_docker-restart-module zz_docker-restart-repeating-reducer zz_docker-restart-sql
-      # These cannot run in parallel, even though the script tries to handle it
-      # TODO: Fix flaky tests https://github.com/clockworklabs/SpacetimeDB/issues/630
-      # - name: Run restarting smoketests
-      #   run: test/run-smoke-tests.sh zz_docker-restart-module zz_docker-restart-repeating-reducer zz_docker-restart-sql
+        run: python -m smoketests --docker
       - name: Stop containers
         if: always()
         run: docker compose down
@@ -31,8 +46,22 @@ jobs:
     name: Test Suite
     runs-on: spacetimedb-runner
     steps:
+      - name: Find Git ref
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          PR_NUMBER="${{ github.event.inputs.pr_number || null }}"
+          if test -n "${PR_NUMBER}"; then
+            GIT_REF="$( gh pr view --repo clockworklabs/SpacetimeDB $PR_NUMBER --json headRefName --jq .headRefName )"
+          else
+            GIT_REF="${{ github.ref }}"
+          fi
+          echo "GIT_REF=${GIT_REF}" >>"$GITHUB_ENV"
+
       - name: Checkout sources
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ env.GIT_REF }}
 
       - uses: dsherret/rust-toolchain-file@v1
 
@@ -46,7 +75,76 @@ jobs:
           sudo chmod 777 /stdb
 
       - name: Run cargo test
-        run: cargo test --all --features odb_rocksdb,odb_sled,metrics
+        run: cargo test --all
+
+      - name: C# bindings tests
+        working-directory: crates/bindings-csharp
+        run: dotnet test -warnaserror
+
+  sdk_test:
+    name: SDK Tests
+    runs-on: spacetimedb-runner
+    steps:
+      - name: Find Git ref
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          PR_NUMBER="${{ github.event.inputs.pr_number || null }}"
+          if test -n "${PR_NUMBER}"; then
+            GIT_REF="$( gh pr view --repo clockworklabs/SpacetimeDB $PR_NUMBER --json headRefName --jq .headRefName )"
+          else
+            GIT_REF="${{ github.ref }}"
+          fi
+          echo "GIT_REF=${GIT_REF}" >>"$GITHUB_ENV"
+
+      - name: Checkout sources
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ env.GIT_REF }}
+
+      - uses: dsherret/rust-toolchain-file@v1
+
+      - uses: actions/setup-dotnet@v3
+        with:
+          dotnet-version: "8.x"
+
+      - name: Create /stdb dir
+        run: |
+          sudo mkdir /stdb
+          sudo chmod 777 /stdb
+      - name: Checkout C# SDK
+        uses: actions/checkout@v4
+        with:
+          repository: clockworklabs/spacetimedb-csharp-sdk
+          path: spacetimedb-csharp-sdk
+
+      - name: C# SDK tests
+        run: |
+          ( cd crates/bindings-csharp/BSATN.Runtime && dotnet pack )
+          cd spacetimedb-csharp-sdk
+
+          # Write out the nuget config file to `nuget.config`. This causes the spacetimedb-csharp-sdk repository
+          # to be aware of the local versions of the `bindings-csharp` packages in SpacetimeDB, and use them if
+          # available. Otherwise, `spacetimedb-csharp-sdk` will use the NuGet versions of the packages.
+          # This means that (if version numbers match) we will test the local versions of the C# packages, even
+          # if they're not pushed to NuGet.
+          # See https://learn.microsoft.com/en-us/nuget/reference/nuget-config-file for more info on the config file,
+          # and https://tldp.org/LDP/abs/html/here-docs.html for more info on this bash feature.
+          cat >nuget.config <<EOF
+          <?xml version="1.0" encoding="utf-8"?>
+          <configuration>
+            <packageSources>
+              <!-- Local NuGet repositories -->
+              <add key="Local SpacetimeDB.BSATN.Runtime" value="../crates/bindings-csharp/BSATN.Runtime/bin/Release" />
+              <!-- Official NuGet.org server -->
+              <add key="NuGet.org" value="https://api.nuget.org/v3/index.json" />
+            </packageSources>
+          </configuration>
+          EOF
+
+          # clear package caches, so we get fresh ones even if version numbers haven't changed
+          dotnet nuget locals all --clear
+          dotnet test
 
   lints:
     name: Lints
@@ -62,7 +160,13 @@ jobs:
         run: cargo fmt --all -- --check
 
       - name: Run cargo clippy
-        run: cargo clippy --all --tests --benches --features odb_rocksdb,odb_sled,metrics -- -D warnings
+        run: cargo clippy --all --tests --benches -- -D warnings
+
+      - name: Run C# formatting check
+        working-directory: crates/bindings-csharp
+        run: |
+          dotnet tool restore
+          dotnet csharpier --check .
 
   wasm_bindings:
     name: Build and test wasm bindings
@@ -74,7 +178,7 @@ jobs:
       - run: echo ::add-matcher::.github/workflows/rust_matcher.json
 
       - name: Build rust-wasm-test
-        run: cargo run -p spacetimedb-cli -- build modules/rust-wasm-test
+        run: cargo run -p spacetimedb-cli -- build --project-path modules/rust-wasm-test
 
       - name: Run bindgen tests
         run: cargo test -p spacetimedb-cli