HOPSFS-264: Update main from upstream up to v1.0.2

.github/CODEOWNERS

@@ -1,7 +1,14 @@
-crates/ @wjones127 @roeap @rtyler @hntd187 @ion-elgreco
-delta-inspect/ @wjones127 @rtyler
+crates/core @roeap @rtyler @hntd187 @ion-elgreco
+crates/deltalake @roeap @rtyler @hntd187 @ion-elgreco
+crates/aws @rtyler
+crates/lakefs @ion-elgreco
+crates/catalog-unity @roeap @hntd187
+
+delta-inspect/ @rtyler
+
 proofs/ @houqp
-python/ @wjones127 @fvaleye @roeap @ion-elgreco
+python/ @wjones127 @roeap @ion-elgreco
+
 tlaplus/ @houqp
-.github/ @wjones127 @rtyler
-docs/ @MrPowers
+
+.github/ @rtyler

.github/actions/setup-env/action.yml

@@ -4,12 +4,12 @@ description: "Set up Python, virtual environment, and Rust toolchain"
 inputs:
   python-version:
     description: "The Python version to set up"
-    required: true
+    required: false
     default: "3.10"
 
   rust-toolchain:
     description: "The Rust toolchain to set up"
-    required: true
+    required: false
     default: "stable"
 
 runs:

.github/codecov.yml

@@ -1,17 +1,18 @@
-
 coverage:
   status:
     project:
       default:
         # allow some leniency on the deviation of pull requests
-        threshold: '1%'
-        informational: true
+        threshold: 1
+        if_ci_failed: error
+        informational: false
     patch:
       default:
-        informational: true
-
+        if_ci_failed: error
+        informational: false
 
 ignore:
   - "delta-inspect/"
   - "proofs/"
   - "**/*.toml"
+  - "crates/benchmarks/"

.github/scripts/retry_integration_test.sh

@@ -5,7 +5,7 @@ MAX_RETRIES=$2
 RETRY_DELAY=$3
 ATTEMPT=1
 run_command() {
-  uv run --no-sync pytest -m "($TEST_NAME and integration)" --doctest-modules 2>&1
+  uv run --no-sync pytest -m "($TEST_NAME and integration and pyarrow)" --doctest-modules 2>&1
 }
 until [ $ATTEMPT -gt $MAX_RETRIES ]
 do

.github/workflows/build.yml

@@ -14,89 +14,80 @@ env:
   # Disable incremental builds by cargo for CI which should save disk space
   # and hopefully avoid final link "No space left on device"
   CARGO_INCREMENTAL: 0
+  SCCACHE_GHA_ENABLED: "true"
+  RUSTC_WRAPPER: "sccache"
 
 jobs:
-  default_build:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-
-      - name: Install minimal stable with clippy and rustfmt
-        uses: actions-rs/toolchain@v1
-        with:
-          profile: default
-          toolchain: '1.82'
-          override: true
-
-      - name: Build
-        run: (cd crates/deltalake && cargo build)
-
-  format:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-
-      - name: Install minimal stable with clippy and rustfmt
-        uses: actions-rs/toolchain@v1
-        with:
-          profile: default
-          toolchain: '1.82'
-          override: true
-
-      - name: Format
-        run: cargo fmt -- --check
-
+  # run various build comnfigurations, fmt, and clippy.
   build:
     strategy:
-      fail-fast: false
+      fail-fast: true
       matrix:
         os:
           - ubuntu-latest
           - windows-latest
+          - macos-latest
     runs-on: ${{ matrix.os }}
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+
+      - name: Run sccache-cache
+        uses: mozilla-actions/sccache-action@v0.0.9
 
       - name: Install minimal stable with clippy and rustfmt
         uses: actions-rs/toolchain@v1
         with:
           profile: default
-          toolchain: '1.82'
+          toolchain: "1.82"
           override: true
 
+      - name: Format
+        run: cargo fmt -- --check
+
+      - name: Default build
+        run: (cd crates/deltalake && cargo build --tests)
+
       - name: build and lint with clippy
         run: cargo clippy --features ${{ env.DEFAULT_FEATURES }} --tests
 
       - name: Spot-check build for native-tls features
         run: cargo clippy --no-default-features --features azure,datafusion,s3-native-tls,gcs,glue --tests
 
-      - name: Check docs
-        run: cargo doc --features ${{ env.DEFAULT_FEATURES }}
-
       - name: Check no default features (except rustls)
         run: cargo check --no-default-features --features rustls
 
-  test:
+      - name: Check docs
+        run: cargo doc --no-deps --features ${{ env.DEFAULT_FEATURES }}
+
+  unit_test:
+    name: Unit Tests
     strategy:
-      fail-fast: false
+      fail-fast: true
       matrix:
         os:
           - ubuntu-latest
           - windows-latest
+          - macos-latest
     runs-on: ${{ matrix.os }}
+
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+
+      - name: Run sccache-cache
+        uses: mozilla-actions/sccache-action@v0.0.9
 
       - name: Install minimal stable with clippy and rustfmt
         uses: actions-rs/toolchain@v1
         with:
           profile: default
-          toolchain: '1.82'
+          toolchain: "1.82"
           override: true
 
       - name: Run tests
-        run: cargo test --verbose --features ${{ env.DEFAULT_FEATURES }}
+        run: |
+          make setup-dat
+          cargo test --features ${{ env.DEFAULT_FEATURES }}
 
   integration_test:
     name: Integration Tests
@@ -115,13 +106,71 @@ jobs:
       AZURE_STORAGE_CONNECTION_STRING: "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://localhost:10000/devstoreaccount1;QueueEndpoint=http://localhost:10001/devstoreaccount1;"
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+
+      - name: Run sccache-cache
+        uses: mozilla-actions/sccache-action@v0.0.9
+
+      - name: Install minimal stable with clippy and rustfmt
+        uses: actions-rs/toolchain@v1
+        with:
+          profile: default
+          toolchain: "1.82"
+          override: true
+
+      - name: Install cargo-llvm-cov
+        uses: taiki-e/install-action@cargo-llvm-cov
+
+      - name: Start emulated services
+        run: docker compose up -d
+
+      - name: Run tests with rustls (default)
+        run: |
+          gmake setup-dat
+          cargo llvm-cov \
+            --features integration_test,${{ env.DEFAULT_FEATURES }} \
+            --workspace \
+            --exclude delta-inspect \
+            --exclude deltalake-hdfs \
+            --exclude deltalake-lakefs \
+            --codecov \
+            --output-path codecov.json
+
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v4
+        with:
+          files: codecov.json
+          fail_ci_if_error: true
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+
+  integration_test_native_tls:
+    name: Integration Tests (Native TLS)
+    runs-on: ubuntu-latest
+    env:
+      # https://github.com/rust-lang/cargo/issues/10280
+      CARGO_NET_GIT_FETCH_WITH_CLI: "true"
+      AWS_DEFAULT_REGION: "us-east-1"
+      AWS_ACCESS_KEY_ID: deltalake
+      AWS_SECRET_ACCESS_KEY: weloverust
+      AWS_ENDPOINT_URL: http://localhost:4566
+      AWS_ALLOW_HTTP: "1"
+      AZURE_USE_EMULATOR: "1"
+      AZURE_STORAGE_ALLOW_HTTP: "1"
+      AZURITE_BLOB_STORAGE_URL: "http://localhost:10000"
+      AZURE_STORAGE_CONNECTION_STRING: "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://localhost:10000/devstoreaccount1;QueueEndpoint=http://localhost:10001/devstoreaccount1;"
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Run sccache-cache
+        uses: mozilla-actions/sccache-action@v0.0.9
 
       - name: Install minimal stable with clippy and rustfmt
         uses: actions-rs/toolchain@v1
         with:
           profile: default
-          toolchain: '1.82'
+          toolchain: "1.82"
           override: true
 
       # Install Java and Hadoop for HDFS integration tests
@@ -139,15 +188,63 @@ jobs:
       - name: Start emulated services
         run: docker compose up -d
 
-      - name: Run tests with rustls (default)
-        run: |
-          cargo test --features integration_test,${{ env.DEFAULT_FEATURES }}
-
       - name: Run tests with native-tls
         run: |
-          cargo clean
+          gmake setup-dat
           cargo test --no-default-features --features integration_test,s3-native-tls,datafusion
 
+  integration_test_hdfs:
+    name: Integration Tests (HDFS)
+    runs-on: ubuntu-latest
+    env:
+      # https://github.com/rust-lang/cargo/issues/10280
+      CARGO_NET_GIT_FETCH_WITH_CLI: "true"
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Run sccache-cache
+        uses: mozilla-actions/sccache-action@v0.0.9
+
+      - name: Install minimal stable with clippy and rustfmt
+        uses: actions-rs/toolchain@v1
+        with:
+          profile: default
+          toolchain: "1.82"
+          override: true
+
+      - name: Install cargo-llvm-cov
+        uses: taiki-e/install-action@cargo-llvm-cov
+
+      # Install Java and Hadoop for HDFS integration tests
+      - uses: actions/setup-java@v4
+        with:
+          distribution: "temurin"
+          java-version: "17"
+
+      - name: Download Hadoop
+        run: |
+          wget -q https://dlcdn.apache.org/hadoop/common/hadoop-3.4.0/hadoop-3.4.0.tar.gz
+          tar -xf hadoop-3.4.0.tar.gz -C $GITHUB_WORKSPACE
+          echo "$GITHUB_WORKSPACE/hadoop-3.4.0/bin" >> $GITHUB_PATH
+
+      - name: Run tests with rustls (default)
+        run: |
+          gmake setup-dat
+          cargo llvm-cov \
+            --features integration_test \
+            --package deltalake-hdfs \
+            --codecov \
+            --output-path codecov.json
+
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v4
+        with:
+          files: codecov.json
+          fail_ci_if_error: true
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+
   integration_test_lakefs:
     name: Integration Tests (LakeFS v1.48)
     runs-on: ubuntu-latest
@@ -156,15 +253,21 @@ jobs:
       CARGO_NET_GIT_FETCH_WITH_CLI: "true"
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+
+      - name: Run sccache-cache
+        uses: mozilla-actions/sccache-action@v0.0.9
 
       - name: Install minimal stable with clippy and rustfmt
         uses: actions-rs/toolchain@v1
         with:
           profile: default
-          toolchain: '1.82'
+          toolchain: "1.82"
           override: true
 
+      - name: Install cargo-llvm-cov
+        uses: taiki-e/install-action@cargo-llvm-cov
+
       - name: Download Lakectl
         run: |
           wget -q https://github.com/treeverse/lakeFS/releases/download/v1.48.1/lakeFS_1.48.1_Linux_x86_64.tar.gz
@@ -176,5 +279,17 @@ jobs:
 
       - name: Run tests with rustls (default)
         run: |
-          cargo test --features integration_test_lakefs,lakefs,datafusion
-
+          gmake setup-dat
+          cargo llvm-cov \
+            --package deltalake-lakefs \
+            --features integration_test_lakefs \
+            --codecov \
+            --output-path codecov.json
+
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v4
+        with:
+          files: codecov.json
+          fail_ci_if_error: true
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}