Update CI to use the source distribution

This makes it possible to detect problems like a missing `.pxd` file. Signed-off-by: Gus Monod <gmonod1@bloomberg.net>
bloomberg · Oct 16, 2023 · ae8d2a5 · ae8d2a5
1 parent 2ba2269
commit ae8d2a5
Show file tree

Hide file tree

Showing 2 changed files with 90 additions and 63 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -1,4 +1,4 @@
-name: Wheels
+name: Tests
 
 on:
   push:
@@ -21,23 +21,16 @@ env:
   TIMEOUT: 30
 
 jobs:
-  build_sdist:
-    name: Build source distribution
+  build_comdb2:
+    name: Build bloomberg/comdb2 from source
     runs-on: ubuntu-latest
     steps:
-      - name: Install dependencies
-        run: '
-          sudo apt-get install -qy
-            pkg-config
-        ' # libcdb2api-dev is installed from source below
-      - uses: actions/checkout@v4
-
-      - name: Checkout comdb2 dependency
-        uses: actions/checkout@v3
+      - name: Checkout bloomberg/comdb2 repository
+        uses: actions/checkout@v4
         with:
           repository: bloomberg/comdb2
-          path: original_comdb2
-      - name: Build comdb2 from source
+          path: bloomberg-comdb2
+      - name: Install build dependencies
         run: '
           sudo apt-get update &&
           sudo apt-get install -qy
@@ -56,22 +49,65 @@ jobs:
             protobuf-c-compiler
             tcl
             uuid-dev
-            zlib1g-dev &&
+            zlib1g-dev
+        '
+      - name: Build from source
+        run: '
           (
-            mkdir original_comdb2/build &&
-            cd original_comdb2/build &&
+            mkdir bloomberg-comdb2/build &&
+            cd bloomberg-comdb2/build &&
             cmake .. &&
-            make &&
-            sudo make install
+            make
           )
         '
+      - name: Archive comdb2 repo with build artifacts
+        run: 'tar czvf bloomberg-comdb2.tar.gz bloomberg-comdb2/'
+      - name: Upload comdb2 repo with build artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: bloomberg-comdb2
+          path: ./bloomberg-comdb2.tar.gz
 
+  build_sdist:
+    name: Build python-comdb2 source distribution
+    runs-on: ubuntu-latest
+    needs: [build_comdb2]
+    steps:
+      - name: Download comdb2 build artifacts
+        uses: actions/download-artifact@v3
+        with:
+          name: bloomberg-comdb2
+          path: .
+      - name: Install bloomberg-comdb2
+        run: '
+          sudo apt-get install -qy
+            libevent-dev
+            liblz4-dev
+            libprotobuf-c-dev
+            libsqlite3-dev
+            libssl-dev
+            libunwind-dev
+            zlib1g-dev &&
+          tar xvf bloomberg-comdb2.tar.gz &&
+          (cd bloomberg-comdb2/build && sudo make install) &&
+          rm -rf bloomberg-comdb2.tar.gz bloomberg-comdb2/
+        '
+      - name: Install python-comdb2 build-time dependencies
+        run: 'sudo apt-get install -qy pkg-config'
+      - uses: actions/checkout@v4
+      - name: 'Set up Python 3.8'
+        uses: actions/setup-python@v4 # note that this step overwrites the PKG_CONFIG_PATH variable
+        with:
+          python-version: '3.8'
       - name: Build sdist
-        run: PKG_CONFIG_PATH=/opt/bb/lib/pkgconfig:/usr/lib/x86_64-linux-gnu/pkgconfig pipx run build --sdist
-
+        run: '
+          PKG_CONFIG_PATH=/opt/bb/lib/pkgconfig:/usr/lib/x86_64-linux-gnu/pkgconfig
+          pipx run build --sdist
+        '
       - name: Upload sdist
         uses: actions/upload-artifact@v3
         with:
+          name: python-comdb2-sdist
           path: dist/*.tar.gz
 
   test:
@@ -87,52 +123,29 @@ jobs:
           - '3.9'
           - '3.8'
     steps:
-      - name: Download sdist
+      - name: Download comdb2 repo with build artifacts
         uses: actions/download-artifact@v3
         with:
-          name: artifact
-          path: dist
-      - name: Install dependencies
-        run: '
-          sudo apt-get install -qy
-            pkg-config
-        ' # libcdb2api-dev is installed from source below
-      - name: Checkout comdb2 dependency
-        uses: actions/checkout@v3
-        with:
-          repository: bloomberg/comdb2
-          path: original_comdb2
-      - name: Build comdb2 from source
+          name: bloomberg-comdb2
+          path: .
+      - name: Install bloomberg-comdb2
         run: '
-          sudo apt-get update &&
           sudo apt-get install -qy
-            bison
-            build-essential
-            cmake
-            flex
             libevent-dev
             liblz4-dev
             libprotobuf-c-dev
-            libreadline-dev
             libsqlite3-dev
             libssl-dev
             libunwind-dev
-            ncurses-dev
-            protobuf-c-compiler
-            tcl
-            uuid-dev
             zlib1g-dev &&
-          (
-            mkdir original_comdb2/build &&
-            cd original_comdb2/build &&
-            cmake .. &&
-            make &&
-            sudo make install
-          )
+          tar xvf bloomberg-comdb2.tar.gz &&
+          (cd bloomberg-comdb2/build && sudo make install) &&
+          rm -rf bloomberg-comdb2.tar.gz bloomberg-comdb2/
         '
       - name: Start local comdb2 instance
         run: '
           sudo mkdir -p /opt/bb/share/schemas/$COMDB2_DBNAME &&
+          sudo mkdir -p /opt/bb/etc/cdb2/config &&
           echo "$COMDB2_DBNAME 1234 $(hostname -f)" > /opt/bb/etc/cdb2/config/comdb2db.cfg &&
           (/opt/bb/bin/pmux -n &) &&
           echo started pmux &&
@@ -150,22 +163,36 @@ jobs:
             fi;
           done
         '
-      - name: Creating tables
+      - name: Download python-comdb2 sdist
+        uses: actions/download-artifact@v3
+        with:
+          name: python-comdb2-sdist
+          path: dist
+      - name: Extract sdist
+        run: '
+          (cd dist/ && tar xvf comdb2-*.tar.gz && rm comdb2-*.tar.gz) &&
+          mv dist/comdb2-* python-comdb2-sdist && rmdir dist
+        '
+      - name: Create tables
         run: |
-          tables="$(cat dist/tests/schemas/$COMDB2_DBNAME/table_constraint_order.txt)"
+          tables="$(cat python-comdb2-sdist/tests/schemas/$COMDB2_DBNAME/table_constraint_order.txt)"
           for table_name in $tables
           do
-            table_file="dist/tests/schemas/$COMDB2_DBNAME/$table_name.csc2"
+            table_file="python-comdb2-sdist/tests/schemas/$COMDB2_DBNAME/$table_name.csc2"
             echo "Creating $table_name from $table_file"
             /opt/bb/bin/cdb2sql "$COMDB2_DBNAME" local "create table $table_name { $(cat $table_file) }"
           done
       - name: Set up Python ${{ matrix.python_version }}
         uses: actions/setup-python@v4 # note that this step overwrites the PKG_CONFIG_PATH variable
         with:
-          python-version: ${{ matrix.python_version }}
-      - name: Install python dependencies
-        run: |
-          python -m pip install --upgrade pip
-          PKG_CONFIG_PATH=/opt/bb/lib/pkgconfig:/usr/lib/x86_64-linux-gnu/pkgconfig LDFLAGS="-Wl,-rpath,/opt/bb/lib" python -m pip install --no-index --find-links=dist/ comdb2[tests]
+          python-version: "${{matrix.python_version}}"
+      - name: Install python-comdb2 from the sdist
+        run: '
+          sudo apt-get install -qy pkg-config &&
+          python -m pip install --upgrade pip &&
+          PKG_CONFIG_PATH=/opt/bb/lib/pkgconfig:/usr/lib/x86_64-linux-gnu/pkgconfig
+          LDFLAGS="-Wl,-rpath,/opt/bb/lib"
+          python -m pip install ./python-comdb2-sdist[tests]
+        '
       - name: Run Tests
-        run: (cd dist/tests && python -m pytest -vvv)
+        run: (cd python-comdb2-sdist/tests && python -m pytest -vvv)
diff --git a/MANIFEST.in b/MANIFEST.in
@@ -1,6 +1,6 @@
 include *.txt
 include pyproject.toml
 recursive-include comdb2/ py.typed *.pyi *.pxd *.h
-recursive-include docs/ *.rst *py
+recursive-include docs/ *.rst *.py
 recursive-include tests/ *.py *.csc2 *.txt
-exclude comdb2/_ccdb2.c
+exclude comdb2/_ccdb2.c*