Merge branch 'master' into popArg-refactor

.github/workflows/build-documentation.yml

@@ -0,0 +1,25 @@
+name: Build documentation
+
+on: [push, pull_request]
+
+jobs:
+
+  build_docs:
+    name: Build RTD documentation
+    runs-on: ubuntu-20.04
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: actions/setup-python@v2
+        name: Install Python
+        with:
+          python-version: '3.9'
+
+      - name: Install documentation dependencies
+        run: |
+          pip install -r docs/requirements.txt
+
+      - name: Build documentation
+        run: |
+          READTHEDOCS=True make -C docs html SPHINXOPTS="-W --keep-going"

.github/workflows/create-palettes.yml

@@ -0,0 +1,51 @@
+name: Generate component palettes
+
+on: [push, pull_request]
+
+jobs:
+
+  run_tests:
+    name: Generate component palettes
+    runs-on: ubuntu-20.04
+    env:
+      PROJECT_NAME: daliuge
+      GIT_REPO: https://github.com/ICRAR/daliuge
+      GITHUB_USERNAME: eagle.updater
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: actions/setup-python@v2
+        name: Install Python
+        with:
+          python-version: '3.9'
+
+      - name: Install system dependencies
+        run: |
+          sudo apt-get update && sudo apt-get install -y doxygen xsltproc
+
+      - name: Configure git
+        run: |
+          git config --global user.name $GITHUB_USERNAME
+          git config --global user.email $GITHUB_USERNAME@gmail.com
+          OUTPUT_FILENAME=$PROJECT_NAME-${GITHUB_REF_NAME/\//_}
+          echo "PROJECT_VERSION=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
+          echo "OUTPUT_FILENAME=$OUTPUT_FILENAME" >> $GITHUB_ENV
+
+      - name: Create palettes
+        run: |
+          python3 tools/xml2palette/xml2palette.py -t daliuge -r ./ $OUTPUT_FILENAME.palette
+          python3 tools/xml2palette/xml2palette.py -t template -r ./ $OUTPUT_FILENAME-template.palette
+
+      - name: Commit palettes to EAGLE
+        env:
+          EAGLE_UPDATER_TOKEN: ${{ secrets.EAGLE_UPDATER_TOKEN }}
+        run: |
+          git clone https://$EAGLE_UPDATER_TOKEN@github.com/ICRAR/EAGLE_test_repo
+          mkdir -p EAGLE_test_repo/$PROJECT_NAME
+          mv $OUTPUT_FILENAME.palette EAGLE_test_repo/$PROJECT_NAME/
+          mv $OUTPUT_FILENAME-template.palette EAGLE_test_repo/$PROJECT_NAME/
+          cd EAGLE_test_repo
+          git add *
+          git diff-index --quiet HEAD || git commit -m "Automatically generated DALiuGE palette (branch $GITHUB_REF_NAME, commit $PROJECT_VERSION)"
+          git push

.github/workflows/linting.yml

@@ -0,0 +1,25 @@
+name: Run pylint
+
+on: [push, pull_request]
+
+jobs:
+
+  pylint:
+    name: Build pylint
+    runs-on: ubuntu-20.04
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: actions/setup-python@v2
+        name: Install Python
+        with:
+          python-version: '3.9'
+
+      - name: Install dependencies
+        run: |
+          pip install pylint
+
+      - name: Run pylint
+        run: |
+          pylint daliuge-common daliuge-translator daliuge-engine

.github/workflows/run-unit-tests.yml

@@ -0,0 +1,88 @@
+name: Run unit tests
+
+on: [push, pull_request]
+
+jobs:
+
+  run_tests:
+    name: Run unit tests with python ${{matrix.python-version}} - ${{ matrix.desc }}
+    runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        include:
+          - python-version: '3.8'
+            test_number: 0
+            engine: no
+            translator: yes
+            desc: "no engine"
+          - python-version: '3.8'
+            test_number: 1
+            desc: "no translator"
+            engine: yes
+            translator: no
+          - python-version: '3.9'
+            test_number: 2
+            desc: "full package"
+            engine: yes
+            translator: yes
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: actions/setup-python@v2
+        name: Install Python
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install test dependencies
+        run: |
+          pip install -U coveralls pytest pytest-cov
+          pip install -U setuptools pip wheel dask dlg-example-cmpts
+
+      - name: Install daliuge-common
+        run: pip install -e daliuge-common/
+
+      - name: Install daliuge-translator
+        if: ${{ matrix.translator == 'yes' }}
+        run: pip install -e daliuge-translator/
+
+      - name: Install daliuge-engine
+        if: ${{ matrix.engine == 'yes' }}
+        run: pip install -e daliuge-engine/
+
+      - name: Run daliuge-translator tests
+        if: ${{ matrix.translator == 'yes' }}
+        run: |
+          COVFILES=" daliuge-translator/.coverage"
+          echo "COVFILES=$COVFILES" >> $GITHUB_ENV
+          cd daliuge-translator
+          pip install -r test-requirements.txt
+          py.test --cov
+
+      - name: Run daliugen-engine tests
+        if: ${{ matrix.engine == 'yes' }}
+        run: |
+          COVFILES="$COVFILES daliuge-engine/.coverage"
+          echo "COVFILES=$COVFILES" >> $GITHUB_ENV
+          cd daliuge-engine
+          py.test --cov
+
+      - name: Combine coverage
+        run: coverage combine $COVFILES
+
+      - name: Update to coveralls
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          COVERALLS_FLAG_NAME: ${{ matrix.test_number }}
+          COVERALLS_PARALLEL: true
+        run: coveralls --service=github
+
+  finish:
+    needs: run_tests
+    runs-on: ubuntu-20.04
+    steps:
+    - name: Coveralls Finished
+      uses: coverallsapp/github-action@master
+      with:
+        github-token: ${{ secrets.GITHUB_TOKEN }}
+        parallel-finished: true

.pylintrc

@@ -0,0 +1,3 @@
+[Main]
+disable=all
+enable=logging-not-lazy,logging-format-interpolation