Showing with 8,259 additions and 2,369 deletions.
  1. +0 −15 .buildkite/build_dozer_tests/Dockerfile
  2. +0 −13 .buildkite/build_dozer_tests/docker-compose.yaml
  3. +0 −6 .buildkite/pipeline.yml
  4. +8 −2 .github/workflows/e2e.yaml
  5. +54 −0 .github/workflows/integration.yaml
  6. +75 −0 .github/workflows/integration/docker-compose.yaml
  7. +4 −0 .github/workflows/integration/dockerfiles/install-curl-ubuntu.sh
  8. +4 −0 .github/workflows/integration/dockerfiles/install-dozer-ubuntu-amd64.sh
  9. +4 −0 .github/workflows/integration/dockerfiles/install-dozer-ubuntu-arm64.sh
  10. +5 −0 .github/workflows/integration/dockerfiles/install-protoc-ubuntu-20-amd64.sh
  11. +5 −0 .github/workflows/integration/dockerfiles/install-protoc-ubuntu-20-arm64.sh
  12. +3 −0 .github/workflows/integration/dockerfiles/install-protoc-ubuntu-22.sh
  13. +13 −0 .github/workflows/integration/dockerfiles/ubuntu-20-amd64
  14. +13 −0 .github/workflows/integration/dockerfiles/ubuntu-20-arm64
  15. +13 −0 .github/workflows/integration/dockerfiles/ubuntu-22-amd64
  16. +13 −0 .github/workflows/integration/dockerfiles/ubuntu-22-arm64
  17. +4 −0 .github/workflows/integration/test-dozer-ubuntu.sh
  18. +10 −0 .github/workflows/integration/test-dozer.sh
  19. +0 −7 .github/workflows/release.yaml
  20. +12 −12 Cargo.lock
  21. +8 −2 README.md
  22. +1 −1 dozer-admin/Cargo.toml
  23. +6 −5 dozer-api/Cargo.toml
  24. +6 −0 dozer-api/README.md
  25. +11 −8 dozer-api/src/api_helper.rs
  26. +31 −0 dozer-api/src/auth/api.rs
  27. +76 −26 dozer-api/src/cache_builder/mod.rs
  28. +13 −2 dozer-api/src/generator/oapi/generator.rs
  29. +28 −0 dozer-api/src/generator/oapi/utils.rs
  30. +32 −19 dozer-api/src/generator/protoc/generator/implementation.rs
  31. +12 −0 dozer-api/src/generator/protoc/generator/mod.rs
  32. +4 −0 dozer-api/src/grpc/auth/mod.rs
  33. +34 −0 dozer-api/src/grpc/auth/service.rs
  34. +17 −13 dozer-api/src/grpc/auth_middleware.rs
  35. +11 −0 dozer-api/src/grpc/client_server.rs
  36. +7 −1 dozer-api/src/grpc/common/service.rs
  37. +1 −0 dozer-api/src/grpc/mod.rs
  38. +6 −4 dozer-api/src/grpc/shared_impl/mod.rs
  39. +18 −3 dozer-api/src/grpc/typed/helper.rs
  40. +6 −2 dozer-api/src/grpc/typed/service.rs
  41. BIN dozer-api/src/grpc/typed/tests/generated_films.bin
  42. +36 −20 dozer-api/src/grpc/types_helper.rs
  43. +19 −4 dozer-api/src/lib.rs
  44. +31 −5 dozer-api/src/rest/api_generator.rs
  45. +4 −10 dozer-api/src/test_utils.rs
  46. +3 −2 dozer-cache/Cargo.toml
  47. +2 −7 dozer-cache/benches/cache.rs
  48. +254 −0 dozer-cache/src/cache/lmdb/cache/main_environment/conflict_resolution_tests.rs
  49. +215 −54 dozer-cache/src/cache/lmdb/cache/main_environment/mod.rs
  50. +26 −17 dozer-cache/src/cache/lmdb/cache/main_environment/operation_log/lmdb_val_impl.rs
  51. +217 −152 dozer-cache/src/cache/lmdb/cache/main_environment/operation_log/mod.rs
  52. +11 −19 dozer-cache/src/cache/lmdb/cache/mod.rs
  53. +3 −3 dozer-cache/src/cache/lmdb/cache/query/handler.rs
  54. +13 −19 dozer-cache/src/cache/lmdb/cache/query/tests.rs
  55. +11 −11 dozer-cache/src/cache/lmdb/cache_manager.rs
  56. +19 −22 dozer-cache/src/cache/lmdb/tests/basic.rs
  57. +1 −2 dozer-cache/src/cache/lmdb/tests/read_write.rs
  58. +1 −4 dozer-cache/src/cache/lmdb/tests/utils.rs
  59. +63 −19 dozer-cache/src/cache/mod.rs
  60. +3 −7 dozer-cache/src/reader.rs
  61. +2 −1 dozer-core/Cargo.toml
  62. +1 −1 dozer-core/src/executor/receiver_loop.rs
  63. +4 −0 dozer-core/src/forwarder.rs
  64. +0 −1 dozer-core/src/tests/dag_base_errors.rs
  65. +0 −3 dozer-core/src/tests/sources.rs
  66. +4 −3 dozer-ingestion/Cargo.toml
  67. +0 −4 dozer-ingestion/SPEC.md
  68. +0 −1 dozer-ingestion/src/connectors/delta_lake/reader.rs
  69. +0 −2 dozer-ingestion/src/connectors/ethereum/log/helper.rs
  70. +0 −1 dozer-ingestion/src/connectors/ethereum/trace/helper.rs
  71. +0 −1 dozer-ingestion/src/connectors/grpc/adapter/default.rs
  72. +0 −4 dozer-ingestion/src/connectors/kafka/debezium/stream_consumer.rs
  73. +65 −28 dozer-ingestion/src/connectors/object_store/table_reader.rs
  74. +25 −3 dozer-ingestion/src/connectors/object_store/tests/local_storage_tests.rs
  75. +1 −1 dozer-ingestion/src/connectors/postgres/helper.rs
  76. +8 −1 dozer-ingestion/src/connectors/postgres/snapshotter.rs
  77. +0 −4 dozer-ingestion/src/connectors/postgres/xlog_mapper.rs
  78. +12 −4 dozer-ingestion/src/connectors/snowflake/connection/client.rs
  79. +2 −3 dozer-ingestion/src/connectors/snowflake/connector/snowflake.rs
  80. +0 −1 dozer-ingestion/src/connectors/snowflake/stream_consumer.rs
  81. +4 −0 dozer-ingestion/src/errors.rs
  82. +2 −2 dozer-ingestion/src/ingestion/ingestor.rs
  83. +12 −5 dozer-ingestion/tests/test_suite/basic.rs
  84. +19 −0 dozer-ingestion/tests/test_suite/connectors/arrow.rs
  85. +6 −0 dozer-ingestion/tests/test_suite/connectors/sql.rs
  86. +3 −3 dozer-orchestrator/Cargo.toml
  87. +0 −2 dozer-orchestrator/src/lib.rs
  88. +2 −1 dozer-orchestrator/src/pipeline/connector_source.rs
  89. +0 −333 dozer-orchestrator/src/pipeline/tests/conflict_resolution_tests.rs
  90. +0 −1 dozer-orchestrator/src/pipeline/tests/mod.rs
  91. +0 −47 dozer-orchestrator/src/test_utils.rs
  92. +5 −4 dozer-sql/Cargo.toml
  93. +111 −47 dozer-sql/src/pipeline/aggregation/avg.rs
  94. +23 −11 dozer-sql/src/pipeline/aggregation/count.rs
  95. +50 −29 dozer-sql/src/pipeline/aggregation/max.rs
  96. +50 −29 dozer-sql/src/pipeline/aggregation/min.rs
  97. +1 −1 dozer-sql/src/pipeline/aggregation/processor.rs
  98. +127 −54 dozer-sql/src/pipeline/aggregation/sum.rs
  99. +237 −6 dozer-sql/src/pipeline/aggregation/tests/aggregation_avg_tests.rs
  100. +196 −5 dozer-sql/src/pipeline/aggregation/tests/aggregation_count_tests.rs
  101. +231 −5 dozer-sql/src/pipeline/aggregation/tests/aggregation_max_tests.rs
  102. +231 −5 dozer-sql/src/pipeline/aggregation/tests/aggregation_min_tests.rs
  103. +1 −2 dozer-sql/src/pipeline/aggregation/tests/aggregation_null.rs
  104. +238 −7 dozer-sql/src/pipeline/aggregation/tests/aggregation_sum_tests.rs
  105. +0 −2 dozer-sql/src/pipeline/aggregation/tests/aggregation_test_planner.rs
  106. +17 −9 dozer-sql/src/pipeline/aggregation/tests/aggregation_tests_utils.rs
  107. +4 −0 dozer-sql/src/pipeline/errors.rs
  108. +38 −2 dozer-sql/src/pipeline/expression/builder.rs
  109. +69 −2 dozer-sql/src/pipeline/expression/cast.rs
  110. +725 −80 dozer-sql/src/pipeline/expression/comparison.rs
  111. +100 −4 dozer-sql/src/pipeline/expression/datetime.rs
  112. +79 −9 dozer-sql/src/pipeline/expression/execution.rs
  113. +107 −33 dozer-sql/src/pipeline/expression/logical.rs
  114. +794 −34 dozer-sql/src/pipeline/expression/mathematical.rs
  115. +3 −0 dozer-sql/src/pipeline/expression/python_udf.rs
  116. +48 −6 dozer-sql/src/pipeline/expression/scalar/number.rs
  117. +42 −3 dozer-sql/src/pipeline/expression/scalar/string.rs
  118. +162 −54 dozer-sql/src/pipeline/expression/tests/cast.rs
  119. +1 −1 dozer-sql/src/pipeline/expression/tests/comparison.rs
  120. +9 −9 dozer-sql/src/pipeline/expression/tests/conditional.rs
  121. +176 −66 dozer-sql/src/pipeline/expression/tests/datetime.rs
  122. +1 −1 dozer-sql/src/pipeline/expression/tests/distance.rs
  123. +12 −7 dozer-sql/src/pipeline/expression/tests/execution.rs
  124. +8 −8 dozer-sql/src/pipeline/expression/tests/logical.rs
  125. +1,133 −55 dozer-sql/src/pipeline/expression/tests/mathematical.rs
  126. +3 −3 dozer-sql/src/pipeline/expression/tests/number.rs
  127. +1 −1 dozer-sql/src/pipeline/expression/tests/point.rs
  128. +4 −4 dozer-sql/src/pipeline/expression/tests/string.rs
  129. +1 −1 dozer-sql/src/pipeline/expression/tests/test_common.rs
  130. +23 −4 dozer-sql/src/pipeline/product/join/factory.rs
  131. +166 −75 dozer-sql/src/pipeline/product/join/operator.rs
  132. +3 −26 dozer-sql/src/pipeline/product/tests/left_join_test.rs
  133. +4 −32 dozer-sql/src/pipeline/product/tests/pipeline_test.rs
  134. +0 −8 dozer-sql/src/pipeline/product/tests/set_operator_test.rs
  135. +4 −4 dozer-sql/src/pipeline/projection/processor.rs
  136. +0 −1 dozer-sql/src/pipeline/tests/builder_test.rs
  137. +0 −5 dozer-sql/src/pipeline/window/tests/operator_test.rs
  138. +0 −9 dozer-sql/src/pipeline/window/tests/pipeline_test.rs
  139. +2 −1 dozer-storage/Cargo.toml
  140. +43 −33 dozer-storage/src/lmdb_database/lmdb_val.rs
  141. +44 −1 dozer-storage/src/lmdb_map.rs
  142. +5 −3 dozer-tests/Cargo.toml
  143. +1 −6 dozer-tests/src/cache_tests/film/load_database.rs
  144. +1 −1 dozer-tests/src/cache_tests/mod.rs
  145. +4 −1 dozer-tests/src/e2e_tests/checker/client.rs
  146. +3 −1 dozer-tests/src/e2e_tests/checker/mod.rs
  147. +1 −6 dozer-tests/src/e2e_tests/runner/local.rs
  148. +3 −1 dozer-tests/src/e2e_tests/runner/mod.rs
  149. +5 −3 dozer-tests/src/sql_tests/helper.rs
  150. +1 −1 dozer-tests/src/sql_tests/mapper.rs
  151. +7 −3 dozer-tests/src/sqllogictest/src/sql_tests/helper.rs
  152. +1 −1 dozer-tests/src/sqllogictest/src/sql_tests/mapper.rs
  153. +2 −19 dozer-tests/src/tests/mapper.rs
  154. +0 −1 dozer-tests/src/tests/sql/python_udf.rs
  155. +2 −1 dozer-tracing/Cargo.toml
  156. +0 −2 dozer-tracing/src/helper.rs
  157. +1 −1 dozer-types/Cargo.toml
  158. +3 −0 dozer-types/build.rs
  159. +26 −0 dozer-types/protos/auth.proto
  160. +1 −1 dozer-types/protos/common.proto
  161. +33 −22 dozer-types/protos/types.proto
  162. +0 −1 dozer-types/src/arrow_types/from_arrow.rs
  163. +15 −2 dozer-types/src/arrow_types/to_arrow.rs
  164. +1 −0 dozer-types/src/borrow/mod.rs
  165. +8 −1 dozer-types/src/field_type.rs
  166. +5 −0 dozer-types/src/grpc_types.rs
  167. +142 −19 dozer-types/src/helper.rs
  168. +9 −0 dozer-types/src/ingestion_types.rs
  169. +3 −3 dozer-types/src/models/api_endpoint.rs
  170. +251 −69 dozer-types/src/types/field.rs
  171. +123 −15 dozer-types/src/types/mod.rs
  172. +595 −404 dozer-types/src/types/tests.rs
  173. +2 −2 dozer-utils/Cargo.toml
  174. +6 −2 dozer-utils/src/process.rs
15 changes: 0 additions & 15 deletions .buildkite/build_dozer_tests/Dockerfile

This file was deleted.

13 changes: 0 additions & 13 deletions .buildkite/build_dozer_tests/docker-compose.yaml

This file was deleted.

6 changes: 0 additions & 6 deletions .buildkite/pipeline.yml

This file was deleted.

10 changes: 8 additions & 2 deletions .github/workflows/e2e.yaml
Expand Up @@ -3,12 +3,14 @@ name: Dozer E2E Test
on:
pull_request_target:
branches: [main, e2e-yaml-dev]
push:
branches: [main, e2e-yaml-dev]

env:
CARGO_TERM_COLOR: always

concurrency:
group: e2e/${{ github.head_ref }}
group: e2e/${{ github.head_ref || github.run_id }}
cancel-in-progress: true

jobs:
Expand All @@ -20,10 +22,14 @@ jobs:
labels: ubuntu-latest-16-cores

steps:
- uses: actions/checkout@v3
- if: github.event_name == 'pull_request_target'
uses: actions/checkout@v3
with:
ref: ${{ github.event.pull_request.head.sha }}

- if: github.event_name == 'push'
uses: actions/checkout@v3

- name: Install rust toolchain
uses: actions-rs/toolchain@v1
with:
Expand Down
54 changes: 54 additions & 0 deletions .github/workflows/integration.yaml
@@ -0,0 +1,54 @@
name: Dozer Integration Test

on:
workflow_dispatch:
inputs:
dozer-version:
description: Expected Dozer version number. Leave blank to skip verifying the version.

env:
CARGO_TERM_COLOR: always
DOZER_VERSION: ${{ github.event.inputs.dozer-version }}

concurrency:
group: integration/${{ github.head_ref }}
cancel-in-progress: true

jobs:
integration-linux:
timeout-minutes: 60
strategy:
matrix:
labels: [ubuntu-latest-16-cores, ubuntu-20-16-cores]
fail-fast: false
runs-on:
labels: ${{ matrix.labels }}
steps:
- uses: actions/checkout@v3

- name: Install Dozer
run: sudo sh .github/workflows/integration/dockerfiles/install-dozer-ubuntu-amd64.sh

- name: Install Protoc Ubuntu 22.04
if: matrix.labels == 'ubuntu-latest-16-cores'
run: sudo sh .github/workflows/integration/dockerfiles/install-protoc-ubuntu-22.sh

- name: Install Protoc Ubuntu 20.04
if: matrix.labels == 'ubuntu-20-16-cores'
run: sudo sh .github/workflows/integration/dockerfiles/install-protoc-ubuntu-20-amd64.sh

- name: Run test
run: sudo sh .github/workflows/integration/test-dozer-ubuntu.sh

integration-macos:
timeout-minutes: 60
runs-on:
labels: macos-12
steps:
- uses: actions/checkout@v3

- name: Install Dozer
run: brew tap getdozer/dozer && brew install dozer

- name: Run test
run: sh .github/workflows/integration/test-dozer.sh
75 changes: 75 additions & 0 deletions .github/workflows/integration/docker-compose.yaml
@@ -0,0 +1,75 @@
version: '3.1'

services:
dozer-tests-ubuntu-20-amd64:
build:
context: dockerfiles
dockerfile: ubuntu-20-amd64
volumes:
- ${PWD}:/dozer
deploy:
resources:
limits:
cpus: '2'
memory: 8G
working_dir: /dozer
environment:
- DOZER_VERSION
command: sh /dozer/.github/workflows/integration/test-dozer-ubuntu.sh
dozer-tests-ubuntu-20-arm64:
build:
context: dockerfiles
dockerfile: ubuntu-20-arm64
volumes:
- ${PWD}:/dozer
deploy:
resources:
limits:
cpus: '2'
memory: 8G
working_dir: /dozer
environment:
- DOZER_VERSION
command: sh /dozer/.github/workflows/integration/test-dozer-ubuntu.sh
dozer-tests-ubuntu-22-amd64:
build:
context: dockerfiles
dockerfile: ubuntu-22-amd64
volumes:
- ${PWD}:/dozer
deploy:
resources:
limits:
cpus: '2'
memory: 8G
working_dir: /dozer
environment:
- DOZER_VERSION
command: sh /dozer/.github/workflows/integration/test-dozer-ubuntu.sh
dozer-tests-ubuntu-22-arm64:
build:
context: dockerfiles
dockerfile: ubuntu-22-arm64
volumes:
- ${PWD}:/dozer
deploy:
resources:
limits:
cpus: '2'
memory: 8G
working_dir: /dozer
environment:
- DOZER_VERSION
command: sh /dozer/.github/workflows/integration/test-dozer-ubuntu.sh
run-tests:
image: alpine
command: echo 'All tests passed'
depends_on:
dozer-tests-ubuntu-20-amd64:
condition: service_completed_successfully
dozer-tests-ubuntu-20-arm64:
condition: service_completed_successfully
dozer-tests-ubuntu-22-amd64:
condition: service_completed_successfully
dozer-tests-ubuntu-22-arm64:
condition: service_completed_successfully
@@ -0,0 +1,4 @@
set -e

apt update
apt install -y curl
@@ -0,0 +1,4 @@
set -e

curl -sLO https://github.com/getdozer/dozer/releases/latest/download/dozer-linux-amd64.deb
dpkg -i dozer-linux-amd64.deb
@@ -0,0 +1,4 @@
set -e

curl -sLO https://github.com/getdozer/dozer/releases/latest/download/dozer-linux-aarch64.deb
dpkg -i dozer-linux-aarch64.deb
@@ -0,0 +1,5 @@
set -e

curl -sLO https://github.com/protocolbuffers/protobuf/releases/download/v22.2/protoc-22.2-linux-x86_64.zip
apt install -y unzip
unzip protoc-22.2-linux-x86_64.zip -d /usr/local
@@ -0,0 +1,5 @@
set -e

curl -sLO https://github.com/protocolbuffers/protobuf/releases/download/v22.2/protoc-22.2-linux-aarch_64.zip
apt install -y unzip
unzip protoc-22.2-linux-aarch_64.zip -d /usr/local
@@ -0,0 +1,3 @@
set -e

apt install -y protobuf-compiler
13 changes: 13 additions & 0 deletions .github/workflows/integration/dockerfiles/ubuntu-20-amd64
@@ -0,0 +1,13 @@
FROM --platform=amd64 ubuntu:20.04

# Install curl
COPY install-curl-ubuntu.sh .
RUN sh install-curl-ubuntu.sh

# Install Protoc
COPY install-protoc-ubuntu-20-amd64.sh .
RUN sh install-protoc-ubuntu-20-amd64.sh

# Install Dozer
COPY install-dozer-ubuntu-amd64.sh .
RUN sh install-dozer-ubuntu-amd64.sh
13 changes: 13 additions & 0 deletions .github/workflows/integration/dockerfiles/ubuntu-20-arm64
@@ -0,0 +1,13 @@
FROM --platform=arm64 ubuntu:20.04

# Install curl
COPY install-curl-ubuntu.sh .
RUN sh install-curl-ubuntu.sh

# Install Protoc
COPY install-protoc-ubuntu-20-arm64.sh .
RUN sh install-protoc-ubuntu-20-arm64.sh

# Install Dozer
COPY install-dozer-ubuntu-arm64.sh .
RUN sh install-dozer-ubuntu-arm64.sh
13 changes: 13 additions & 0 deletions .github/workflows/integration/dockerfiles/ubuntu-22-amd64
@@ -0,0 +1,13 @@
FROM --platform=amd64 ubuntu:22.04

# Install curl
COPY install-curl-ubuntu.sh .
RUN sh install-curl-ubuntu.sh

# Install Protoc
COPY install-protoc-ubuntu-22.sh .
RUN sh install-protoc-ubuntu-22.sh

# Install Dozer
COPY install-dozer-ubuntu-amd64.sh .
RUN sh install-dozer-ubuntu-amd64.sh
13 changes: 13 additions & 0 deletions .github/workflows/integration/dockerfiles/ubuntu-22-arm64
@@ -0,0 +1,13 @@
FROM --platform=arm64 ubuntu:22.04

# Install curl
COPY install-curl-ubuntu.sh .
RUN sh install-curl-ubuntu.sh

# Install Protoc
COPY install-protoc-ubuntu-22.sh .
RUN sh install-protoc-ubuntu-22.sh

# Install Dozer
COPY install-dozer-ubuntu-arm64.sh .
RUN sh install-dozer-ubuntu-arm64.sh
4 changes: 4 additions & 0 deletions .github/workflows/integration/test-dozer-ubuntu.sh
@@ -0,0 +1,4 @@
set -e

apt install -y build-essential
sh .github/workflows/integration/test-dozer.sh
10 changes: 10 additions & 0 deletions .github/workflows/integration/test-dozer.sh
@@ -0,0 +1,10 @@
set -e

# Check if dozer version matches `DOZER_VERSION`
dozer -V | grep "$DOZER_VERSION"

# Install Rust
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y

# Run grpc ingest test because it doesn't need docker or ETH secrets.
CARGO_TARGET_DIR=../ DOZER_BIN=dozer RUST_LOG=info "$HOME/.cargo/bin/cargo" run -p dozer-tests --bin dozer-tests -- grpc_ingest
7 changes: 0 additions & 7 deletions .github/workflows/release.yaml
Expand Up @@ -392,13 +392,6 @@ jobs:
docker tag $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG $ECR_REGISTRY/$ECR_REPOSITORY:latest
docker push $ECR_REGISTRY/$ECR_REPOSITORY:latest
- name: Trigger BuildKite integration tests
if: matrix.os == 'ubuntu-20-16-cores'
run: |
curl -X POST "https://api.buildkite.com/v2/organizations/dozer-data-pte-ltd/pipelines/integration-tests/builds" \
--header 'Authorization: Bearer ${{ secrets.BUILDKITE_API_ACCESS_TOKEN_WRITE_BUILDS }}' \
-d '{ "commit": "${{ github.sha }}", "branch": "${{ github.ref_name }}", "env": { "DOZER_VERSION": "${{ github.sha }}" }, "ignore_pipeline_branch_filters": true }'
- name: Release notification
if: ${{ env.VERSION != 'dev' && matrix.os == 'ubuntu-20-16-cores'}}
env:
Expand Down