From 57e5ef0fc4c647c2b84f3edb05df2302aa1384f0 Mon Sep 17 00:00:00 2001 From: Jixiang Jiang Date: Thu, 17 Mar 2022 23:40:39 -0400 Subject: [PATCH] musl only for debugging --- .github/workflows/main.yml | 554 ++++++++++++++--------------- crates/connector_proxy/src/main.rs | 25 +- 2 files changed, 290 insertions(+), 289 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d412c4c1aa..5130005a16 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -14,87 +14,87 @@ env: CARGO_SQLX_VERSION: 0.5.10 jobs: - flowctl: - runs-on: ubuntu-20.04 - env: - SQLX_OFFLINE: true - - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 0 - submodules: true - - name: Install protobuf compiler (it's not already included in CI runner) - run: sudo apt install -y libprotobuf-dev protobuf-compiler - - # This weird installation method is taken from this comment: - # https://github.com/actions-rs/toolchain/issues/126#issuecomment-782989659 - # Also: https://github.com/rust-lang/rustup/issues/2070#issuecomment-545096849 - - name: Install rust toolchain - run: rustup show - - # We require a minimal Go version of 1.17. - - uses: actions/setup-go@v2 - with: - go-version: "1.17.3" - - name: Set up Google Cloud SDK - uses: google-github-actions/setup-gcloud@master - with: - project_id: ${{ secrets.GCP_PROJECT_ID }} - service_account_key: ${{ secrets.GCP_SA_KEY }} - export_default_credentials: true - - - run: make extra-ci-runner-setup - - run: make print-versions - - - name: Login to GitHub container registry - run: | - echo "${{ secrets.GITHUB_TOKEN }}" | \ - docker login --username ${{ github.actor }} --password-stdin ghcr.io - - - name: Cache/Restore Rust dependencies. - uses: actions/cache@v2 - with: - # See: https://doc.rust-lang.org/nightly/cargo/guide/cargo-home.html#caching-the-cargo-home-in-ci - # TODO: Try https://github.com/Swatinem/rust-cache - path: | - ~/.cargo/registry/cache - ~/.cargo/registry/index - ~/.cargo/git/db - ~/.cargo/bin - target - key: ${{ runner.os }}-glibc-cargo-${{ hashFiles('**/Cargo.lock') }} - restore-keys: | - ${{ runner.os }}-glibc-cargo- - - - name: Cache/Restore Go dependencies. - uses: actions/cache@v2 - with: - path: ~/go/pkg/mod - key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} - - # Build and test all the things. - - run: make install-tools - - run: go mod download - - run: make rust-test - - run: make rust-binaries - - - name: Upload Binaries - uses: actions/upload-artifact@v2 - with: - name: flowctl_binaries - path: | - .build/package/bin/* - - # Use to the cached version of cargo-cache if it exists. Saves minutes installing it. - - run: | - if [[ $(cargo cache --version) != "cargo-cache $CARGO_CACHE_VERSION" ]]; then - cargo install --force cargo-cache --version="$CARGO_CACHE_VERSION" - fi - shell: bash - - # Cleanup in preparation for populating the build cache. - - run: cargo cache --autoclean + #flowctl: + # runs-on: ubuntu-20.04 + # env: + # SQLX_OFFLINE: true + + # steps: + # - uses: actions/checkout@v2 + # with: + # fetch-depth: 0 + # submodules: true + # - name: Install protobuf compiler (it's not already included in CI runner) + # run: sudo apt install -y libprotobuf-dev protobuf-compiler + + # # This weird installation method is taken from this comment: + # # https://github.com/actions-rs/toolchain/issues/126#issuecomment-782989659 + # # Also: https://github.com/rust-lang/rustup/issues/2070#issuecomment-545096849 + # - name: Install rust toolchain + # run: rustup show + + # # We require a minimal Go version of 1.17. + # - uses: actions/setup-go@v2 + # with: + # go-version: "1.17.3" + # - name: Set up Google Cloud SDK + # uses: google-github-actions/setup-gcloud@master + # with: + # project_id: ${{ secrets.GCP_PROJECT_ID }} + # service_account_key: ${{ secrets.GCP_SA_KEY }} + # export_default_credentials: true + + # - run: make extra-ci-runner-setup + # - run: make print-versions + + # - name: Login to GitHub container registry + # run: | + # echo "${{ secrets.GITHUB_TOKEN }}" | \ + # docker login --username ${{ github.actor }} --password-stdin ghcr.io + + # - name: Cache/Restore Rust dependencies. + # uses: actions/cache@v2 + # with: + # # See: https://doc.rust-lang.org/nightly/cargo/guide/cargo-home.html#caching-the-cargo-home-in-ci + # # TODO: Try https://github.com/Swatinem/rust-cache + # path: | + # ~/.cargo/registry/cache + # ~/.cargo/registry/index + # ~/.cargo/git/db + # ~/.cargo/bin + # target + # key: ${{ runner.os }}-glibc-cargo-${{ hashFiles('**/Cargo.lock') }} + # restore-keys: | + # ${{ runner.os }}-glibc-cargo- + + # - name: Cache/Restore Go dependencies. + # uses: actions/cache@v2 + # with: + # path: ~/go/pkg/mod + # key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + + # # Build and test all the things. + # - run: make install-tools + # - run: go mod download + # - run: make rust-test + # - run: make rust-binaries + + # - name: Upload Binaries + # uses: actions/upload-artifact@v2 + # with: + # name: flowctl_binaries + # path: | + # .build/package/bin/* + + # # Use to the cached version of cargo-cache if it exists. Saves minutes installing it. + # - run: | + # if [[ $(cargo cache --version) != "cargo-cache $CARGO_CACHE_VERSION" ]]; then + # cargo install --force cargo-cache --version="$CARGO_CACHE_VERSION" + # fi + # shell: bash + + # # Cleanup in preparation for populating the build cache. + # - run: cargo cache --autoclean musl: runs-on: ubuntu-20.04 @@ -139,7 +139,7 @@ jobs: # Build and test all the things. - run: go mod download - - run: make musl-test + #- run: make musl-test - run: make musl-binaries - name: Upload Binaries @@ -159,198 +159,198 @@ jobs: # Cleanup in preparation for populating the build cache. - run: cargo cache --autoclean - stage2: - runs-on: ubuntu-20.04 - needs: [flowctl, musl] - env: - DATABASE_URL: postgres://flow:flow@localhost:5432/control_template - - services: - postgres: - image: postgres:14 - env: - # POSTGRES_HOST: When running jobs on a runner machine (as opposed to - # within Docker), the postgres instance is exposed on localhost. - POSTGRES_PORT: 5432 - POSTGRES_USER: flow - POSTGRES_PASSWORD: flow - POSTGRES_DB: control_template - ports: - - 5432:5432 - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 0 - submodules: true - - - name: Install protobuf compiler (it's not already included in CI runner) - run: sudo apt install -y libprotobuf-dev protobuf-compiler - - - name: Install rust toolchain - run: rustup show - - run: make extra-ci-runner-setup - - run: make print-versions - - - name: Set up Google Cloud SDK - uses: google-github-actions/setup-gcloud@master - with: - project_id: ${{ secrets.GCP_PROJECT_ID }} - service_account_key: ${{ secrets.GCP_SA_KEY }} - export_default_credentials: true - - - name: Login to GitHub container registry - run: | - echo "${{ secrets.GITHUB_TOKEN }}" | \ - docker login --username ${{ github.actor }} --password-stdin ghcr.io - - # We still need to be able to build Rust proto files, even in this stage. - - run: go mod download - - - name: Download Stage1 Binaries - uses: actions/download-artifact@v2 - with: - path: artifacts - - - name: Move Artifacts to PKGBIN - run: | - mkdir -p .build/package/bin && \ - mv artifacts/*/* .build/package/bin - - # Downloaded artifacts lose their prior permission settings - - name: Adjust stage1_binaries permissions - run: | - chmod +x .build/package/bin/* - - - name: Add PKGDIR/bin to PATH - run: echo "$GITHUB_WORKSPACE/.build/package/bin" >> $GITHUB_PATH - - - name: Cache/Restore Rust dependencies. - uses: actions/cache@v2 - with: - path: | - ~/.cargo/registry/cache - ~/.cargo/registry/index - ~/.cargo/git/db - ~/.cargo/bin - target - key: ${{ runner.os }}-glibc-cargo-${{ hashFiles('**/Cargo.lock') }} - restore-keys: | - ${{ runner.os }}-glibc-cargo- - - # Use to the cached version of sqlx if it exists. Saves minutes installing it. - - run: | - if [[ $(cargo sqlx --version) != "cargo-sqlx $CARGO_SQLX_VERSION" ]]; then - cargo install sqlx-cli --version="$CARGO_SQLX_VERSION" - fi - shell: bash - - # Run these checks and tests from the crates/control directory to work - # around poor sqlx support for workspaces and to prevent recompiling many - # crates over again due to conflicting feature resolution. - - name: Setup Control Database - run: cargo sqlx database setup - working-directory: ./crates/control - - name: Verify SQLx Query Cache - run: cargo sqlx prepare --check -- --lib --release --all-features - working-directory: ./crates/control - - name: Control Tests - run: cargo test --lib --test it --release --all-features - working-directory: ./crates/control - - # Use make's `-o` flag to mark the binary artifacts they depend on as - # "very old", which prevents Make from attempting to rebuild them from - # scratch. This prevents us from recompiling them again here. - - name: make catalog-test - run: | - make catalog-test \ - -o /home/runner/work/flow/flow/.build/package/bin/etcd \ - -o /home/runner/work/flow/flow/.build/package/bin/flowctl \ - -o /home/runner/work/flow/flow/.build/package/bin/flowctl-go \ - -o /home/runner/work/flow/flow/.build/package/bin/flow-connector-proxy \ - -o /home/runner/work/flow/flow/.build/package/bin/flow-network-proxy \ - -o /home/runner/work/flow/flow/.build/package/bin/flow-parser \ - -o /home/runner/work/flow/flow/.build/package/bin/flow-schemalate \ - -o /home/runner/work/flow/flow/.build/package/bin/gazette \ - -o /home/runner/work/flow/flow/.build/package/bin/sops - - - name: make go-test-ci - run: | - make go-test-ci \ - -o /home/runner/work/flow/flow/.build/package/bin/etcd \ - -o /home/runner/work/flow/flow/.build/package/bin/flowctl \ - -o /home/runner/work/flow/flow/.build/package/bin/flowctl-go \ - -o /home/runner/work/flow/flow/.build/package/bin/flow-connector-proxy \ - -o /home/runner/work/flow/flow/.build/package/bin/flow-network-proxy \ - -o /home/runner/work/flow/flow/.build/package/bin/flow-parser \ - -o /home/runner/work/flow/flow/.build/package/bin/flow-schemalate \ - -o /home/runner/work/flow/flow/.build/package/bin/gazette \ - -o /home/runner/work/flow/flow/.build/package/bin/sops - - - name: make end-to-end-test - run: | - make end-to-end-test \ - -o /home/runner/work/flow/flow/.build/package/bin/etcd \ - -o /home/runner/work/flow/flow/.build/package/bin/flowctl \ - -o /home/runner/work/flow/flow/.build/package/bin/flowctl-go \ - -o /home/runner/work/flow/flow/.build/package/bin/flow-connector-proxy \ - -o /home/runner/work/flow/flow/.build/package/bin/flow-network-proxy \ - -o /home/runner/work/flow/flow/.build/package/bin/flow-parser \ - -o /home/runner/work/flow/flow/.build/package/bin/flow-schemalate \ - -o /home/runner/work/flow/flow/.build/package/bin/gazette \ - -o /home/runner/work/flow/flow/.build/package/bin/sops - - - name: make package (tar only) - run: | - make /home/runner/work/flow/flow/.build/package/flow-x86-linux.tar.gz \ - -o /home/runner/work/flow/flow/.build/package/bin/etcd \ - -o /home/runner/work/flow/flow/.build/package/bin/flowctl \ - -o /home/runner/work/flow/flow/.build/package/bin/flowctl-go \ - -o /home/runner/work/flow/flow/.build/package/bin/flow-connector-proxy \ - -o /home/runner/work/flow/flow/.build/package/bin/flow-network-proxy \ - -o /home/runner/work/flow/flow/.build/package/bin/flow-parser \ - -o /home/runner/work/flow/flow/.build/package/bin/flow-schemalate \ - -o /home/runner/work/flow/flow/.build/package/bin/gazette \ - -o /home/runner/work/flow/flow/.build/package/bin/sops - - - name: Tidy up, and ensure that generated files are unchanged. - run: | - # catalog-test dirties the branch by creating examples/examples.db. - git clean --force -d - git diff - [[ -z "$(git status --porcelain)" ]] || exit 1 - shell: bash - - # Package and push images. - - run: make docker-image - - run: make docker-push - - if: ${{ github.ref == 'refs/heads/master' }} - run: make docker-push-dev - - - if: ${{ github.ref == 'refs/heads/master' }} - name: Upload release assets - uses: softprops/action-gh-release@v1 - with: - body: | - This is the latest development build, which gets updated on every successful build of the master branch. - target_commitish: ${{ github.sha }} - draft: false - prerelease: true - tag_name: dev - generate_release_notes: false - files: .build/package/flow-x86-linux.tar.gz - - # Use to the cached version of cargo-cache if it exists. Saves minutes installing it. - - run: | - if [[ $(cargo cache --version) != "cargo-cache $CARGO_CACHE_VERSION" ]]; then - cargo install --force cargo-cache --version="$CARGO_CACHE_VERSION" - fi - shell: bash - - # Cleanup in preparation for populating the build cache. - - run: cargo cache --autoclean + #stage2: + # runs-on: ubuntu-20.04 + # needs: [flowctl, musl] + # env: + # DATABASE_URL: postgres://flow:flow@localhost:5432/control_template + + # services: + # postgres: + # image: postgres:14 + # env: + # # POSTGRES_HOST: When running jobs on a runner machine (as opposed to + # # within Docker), the postgres instance is exposed on localhost. + # POSTGRES_PORT: 5432 + # POSTGRES_USER: flow + # POSTGRES_PASSWORD: flow + # POSTGRES_DB: control_template + # ports: + # - 5432:5432 + # options: >- + # --health-cmd pg_isready + # --health-interval 10s + # --health-timeout 5s + # --health-retries 5 + + # steps: + # - uses: actions/checkout@v2 + # with: + # fetch-depth: 0 + # submodules: true + + # - name: Install protobuf compiler (it's not already included in CI runner) + # run: sudo apt install -y libprotobuf-dev protobuf-compiler + + # - name: Install rust toolchain + # run: rustup show + # - run: make extra-ci-runner-setup + # - run: make print-versions + + # - name: Set up Google Cloud SDK + # uses: google-github-actions/setup-gcloud@master + # with: + # project_id: ${{ secrets.GCP_PROJECT_ID }} + # service_account_key: ${{ secrets.GCP_SA_KEY }} + # export_default_credentials: true + + # - name: Login to GitHub container registry + # run: | + # echo "${{ secrets.GITHUB_TOKEN }}" | \ + # docker login --username ${{ github.actor }} --password-stdin ghcr.io + + # # We still need to be able to build Rust proto files, even in this stage. + # - run: go mod download + + # - name: Download Stage1 Binaries + # uses: actions/download-artifact@v2 + # with: + # path: artifacts + + # - name: Move Artifacts to PKGBIN + # run: | + # mkdir -p .build/package/bin && \ + # mv artifacts/*/* .build/package/bin + + # # Downloaded artifacts lose their prior permission settings + # - name: Adjust stage1_binaries permissions + # run: | + # chmod +x .build/package/bin/* + + # - name: Add PKGDIR/bin to PATH + # run: echo "$GITHUB_WORKSPACE/.build/package/bin" >> $GITHUB_PATH + + # - name: Cache/Restore Rust dependencies. + # uses: actions/cache@v2 + # with: + # path: | + # ~/.cargo/registry/cache + # ~/.cargo/registry/index + # ~/.cargo/git/db + # ~/.cargo/bin + # target + # key: ${{ runner.os }}-glibc-cargo-${{ hashFiles('**/Cargo.lock') }} + # restore-keys: | + # ${{ runner.os }}-glibc-cargo- + + # # Use to the cached version of sqlx if it exists. Saves minutes installing it. + # - run: | + # if [[ $(cargo sqlx --version) != "cargo-sqlx $CARGO_SQLX_VERSION" ]]; then + # cargo install sqlx-cli --version="$CARGO_SQLX_VERSION" + # fi + # shell: bash + + # # Run these checks and tests from the crates/control directory to work + # # around poor sqlx support for workspaces and to prevent recompiling many + # # crates over again due to conflicting feature resolution. + # - name: Setup Control Database + # run: cargo sqlx database setup + # working-directory: ./crates/control + # - name: Verify SQLx Query Cache + # run: cargo sqlx prepare --check -- --lib --release --all-features + # working-directory: ./crates/control + # - name: Control Tests + # run: cargo test --lib --test it --release --all-features + # working-directory: ./crates/control + + # # Use make's `-o` flag to mark the binary artifacts they depend on as + # # "very old", which prevents Make from attempting to rebuild them from + # # scratch. This prevents us from recompiling them again here. + # - name: make catalog-test + # run: | + # make catalog-test \ + # -o /home/runner/work/flow/flow/.build/package/bin/etcd \ + # -o /home/runner/work/flow/flow/.build/package/bin/flowctl \ + # -o /home/runner/work/flow/flow/.build/package/bin/flowctl-go \ + # -o /home/runner/work/flow/flow/.build/package/bin/flow-connector-proxy \ + # -o /home/runner/work/flow/flow/.build/package/bin/flow-network-proxy \ + # -o /home/runner/work/flow/flow/.build/package/bin/flow-parser \ + # -o /home/runner/work/flow/flow/.build/package/bin/flow-schemalate \ + # -o /home/runner/work/flow/flow/.build/package/bin/gazette \ + # -o /home/runner/work/flow/flow/.build/package/bin/sops + + # - name: make go-test-ci + # run: | + # make go-test-ci \ + # -o /home/runner/work/flow/flow/.build/package/bin/etcd \ + # -o /home/runner/work/flow/flow/.build/package/bin/flowctl \ + # -o /home/runner/work/flow/flow/.build/package/bin/flowctl-go \ + # -o /home/runner/work/flow/flow/.build/package/bin/flow-connector-proxy \ + # -o /home/runner/work/flow/flow/.build/package/bin/flow-network-proxy \ + # -o /home/runner/work/flow/flow/.build/package/bin/flow-parser \ + # -o /home/runner/work/flow/flow/.build/package/bin/flow-schemalate \ + # -o /home/runner/work/flow/flow/.build/package/bin/gazette \ + # -o /home/runner/work/flow/flow/.build/package/bin/sops + + # - name: make end-to-end-test + # run: | + # make end-to-end-test \ + # -o /home/runner/work/flow/flow/.build/package/bin/etcd \ + # -o /home/runner/work/flow/flow/.build/package/bin/flowctl \ + # -o /home/runner/work/flow/flow/.build/package/bin/flowctl-go \ + # -o /home/runner/work/flow/flow/.build/package/bin/flow-connector-proxy \ + # -o /home/runner/work/flow/flow/.build/package/bin/flow-network-proxy \ + # -o /home/runner/work/flow/flow/.build/package/bin/flow-parser \ + # -o /home/runner/work/flow/flow/.build/package/bin/flow-schemalate \ + # -o /home/runner/work/flow/flow/.build/package/bin/gazette \ + # -o /home/runner/work/flow/flow/.build/package/bin/sops + + # - name: make package (tar only) + # run: | + # make /home/runner/work/flow/flow/.build/package/flow-x86-linux.tar.gz \ + # -o /home/runner/work/flow/flow/.build/package/bin/etcd \ + # -o /home/runner/work/flow/flow/.build/package/bin/flowctl \ + # -o /home/runner/work/flow/flow/.build/package/bin/flowctl-go \ + # -o /home/runner/work/flow/flow/.build/package/bin/flow-connector-proxy \ + # -o /home/runner/work/flow/flow/.build/package/bin/flow-network-proxy \ + # -o /home/runner/work/flow/flow/.build/package/bin/flow-parser \ + # -o /home/runner/work/flow/flow/.build/package/bin/flow-schemalate \ + # -o /home/runner/work/flow/flow/.build/package/bin/gazette \ + # -o /home/runner/work/flow/flow/.build/package/bin/sops + + # - name: Tidy up, and ensure that generated files are unchanged. + # run: | + # # catalog-test dirties the branch by creating examples/examples.db. + # git clean --force -d + # git diff + # [[ -z "$(git status --porcelain)" ]] || exit 1 + # shell: bash + + # # Package and push images. + # - run: make docker-image + # - run: make docker-push + # - if: ${{ github.ref == 'refs/heads/master' }} + # run: make docker-push-dev + + # - if: ${{ github.ref == 'refs/heads/master' }} + # name: Upload release assets + # uses: softprops/action-gh-release@v1 + # with: + # body: | + # This is the latest development build, which gets updated on every successful build of the master branch. + # target_commitish: ${{ github.sha }} + # draft: false + # prerelease: true + # tag_name: dev + # generate_release_notes: false + # files: .build/package/flow-x86-linux.tar.gz + + # # Use to the cached version of cargo-cache if it exists. Saves minutes installing it. + # - run: | + # if [[ $(cargo cache --version) != "cargo-cache $CARGO_CACHE_VERSION" ]]; then + # cargo install --force cargo-cache --version="$CARGO_CACHE_VERSION" + # fi + # shell: bash + + # # Cleanup in preparation for populating the build cache. + # - run: cargo cache --autoclean diff --git a/crates/connector_proxy/src/main.rs b/crates/connector_proxy/src/main.rs index a5c6383a87..f4c6d7fb38 100644 --- a/crates/connector_proxy/src/main.rs +++ b/crates/connector_proxy/src/main.rs @@ -96,18 +96,19 @@ static DEFAULT_CONNECTOR_ENTRYPOINT: &str = "/connector/connector"; #[tokio::main] async fn main() -> std::io::Result<()> { - let Args { - image_inspect_json_path, - proxy_command, - log_args, - } = Args::parse(); - init_logging(&log_args); - - let result = async_main(image_inspect_json_path, proxy_command).await; - if let Err(err) = result.as_ref() { - tracing::error!("connector proxy execution failed. {:?}", err); - std::process::exit(1); - } + println!("hello world"); + //let Args { + // image_inspect_json_path, + // proxy_command, + // log_args, + //} = Args::parse(); + //init_logging(&log_args); + + //let result = async_main(image_inspect_json_path, proxy_command).await; + //if let Err(err) = result.as_ref() { + // tracing::error!("connector proxy execution failed. {:?}", err); + // std::process::exit(1); + //} Ok(()) }