Add code coverage feature

.github/workflows/aws-coverage.yml

@@ -0,0 +1,56 @@
+# This workflow will run an aws backend test.
+name: AWS coverage
+on:
+  push:
+    branches: [ "add-code-coverage" ]
+    paths:
+      - 'lithops/serverless/**'
+      - '!lithops/serverless/backends/k8s/**'
+      - '.github/workflows/aws-coverage.yml'
+  pull_request:
+    branches: [ "add-code-coverage" ]
+    paths:
+      - 'lithops/serverless/**'
+      - '!lithops/serverless/backends/k8s/**'
+      - '.github/workflows/aws-coverage.yml'
+jobs:
+  build_aws:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10"]
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install pytest
+        python -m pip install pytest-cov
+    - name: Install lithops
+      run: |
+        pip install .
+    - name: Test aws with pytest
+      run: |
+        echo "lithops:" > .lithops_config
+        echo "  backend: aws_lambda" >> .lithops_config
+        echo "  storage: aws_s3" >> .lithops_config
+        echo "aws:" >> .lithops_config
+        echo "  region: us-east-1" >> .lithops_config
+        echo "  access_key_id: ${{ secrets.AWS_KEY_ID }}" >> .lithops_config
+        echo "  secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}" >> .lithops_config
+        echo "aws_lambda:" >> .lithops_config
+        echo "  execution_role: ${{ secrets.AWS_ROLE }}" >> .lithops_config
+        coverage run --omit=./lithops/standalone/*,./lithops/util/*,./lithops/libs/*,./lithops/localhost/*,./lithops/utils.py,./lithops/tests/util_func/*,./lithops/scripts/**,./lithops/tests/test_template.py,./lithops/tests/tests_main.py -m pytest ./lithops/tests/github_coverage.py -k aws
+        coverage xml
+    - name: Upload coverage to Codecov
+      uses: codecov/codecov-action@v4.0.1
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}
+        verbose: true
+        slug: acanadil/lithops
+        flags: aws

.github/workflows/k8s-coverage.yml

@@ -0,0 +1,100 @@
+# This workflow will run a k8s backend.
+name: K8s coverage
+on:
+  push:
+    branches: [ "add-code-coverage" ]
+    paths:
+      - 'lithops/serverless/**'
+      - '!lithops/serverless/backends/aws_lambda/**'
+      - '.github/workflows/k8s-coverage.yml'
+      - 'lithops/storage/**'
+  pull_request:
+    branches: [ "add-code-coverage" ]
+    paths:
+      - 'lithops/serverless/**'
+      - '!lithops/serverless/backends/aws_lambda/**'
+      - '.github/workflows/k8s-coverage.yml'
+      - 'lithops/storage/**'
+jobs:
+  build_k8s:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10"]
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install pytest
+        python -m pip install pytest-cov
+    - name: Install lithops
+      run: |
+        pip install .
+    - name: Install docker
+      run: |
+        sudo apt-get update
+        sudo apt-get install ca-certificates curl
+        sudo install -m 0755 -d /etc/apt/keyrings
+        sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc
+        sudo chmod a+r /etc/apt/keyrings/docker.asc
+        echo \
+          "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \
+          $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \
+          sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
+        sudo apt-get update
+        sudo apt-get install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
+        sudo docker run hello-world
+        sudo usermod -aG docker $USER
+        docker login -u ${{ secrets.DOCKER_USER }} -p ${{ secrets.DOCKER_TOKEN }}
+    - name: Install minikube
+      run: |
+        curl -LO https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64
+        sudo install minikube-linux-amd64 /usr/local/bin/minikube
+        minikube start
+    - name: Install kubectl
+      run: |
+        curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
+        sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
+    - name: Wait until aws-coverage finishes
+      run: |
+        status="in_progress"
+        check="completed"
+        while [ "$check" != "$status" ]; do
+          response=$(curl -H "Accept: application/vnd.github.v3+json" "https://api.github.com/repos/$GITHUB_REPOSITORY/actions/runs?&head_sha=$GITHUB_SHA")
+          status=$(echo "$response" | jq -r '.workflow_runs[] | select(.path == ".github/workflows/aws-coverage.yml") | .status')
+          if [ "$status" == "" ]; then
+            status="completed"
+          fi
+          sleep 30
+        done
+    - name: Test k8s with pytest
+      run: |
+        echo "lithops:" > .lithops_config
+        echo "  backend: k8s" >> .lithops_config
+        echo "  storage: aws_s3" >> .lithops_config
+        echo "k8s:" >> .lithops_config
+        echo "  docker_server: docker.io" >> .lithops_config
+        echo "  docker_user: ${{ secrets.DOCKER_USER }}" >> .lithops_config
+        echo "  docker_password: ${{ secrets.DOCKER_TOKEN }}" >> .lithops_config
+        echo "  runtime: ${{ secrets.DOCKER_RUNTIME }}" >> .lithops_config
+        echo "  runtime_memory: 500" >> .lithops_config
+        echo "  runtime_cpu: 1" >> .lithops_config
+        echo "aws:" >> .lithops_config
+        echo "  region: us-east-1" >> .lithops_config
+        echo "  access_key_id: ${{ secrets.AWS_KEY_ID }}" >> .lithops_config
+        echo "  secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}" >> .lithops_config
+        coverage run --omit=./lithops/standalone/*,./lithops/util/*,./lithops/libs/*,./lithops/localhost/*,./lithops/utils.py,./lithops/tests/util_func/*,./lithops/scripts/**,./lithops/tests/test_template.py,./lithops/tests/tests_main.py -m pytest ./lithops/tests/github_coverage.py -k k8s
+        coverage xml   
+    - name: Upload coverage to Codecov
+      uses: codecov/codecov-action@v4.0.1
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}
+        verbose: true
+        slug: acanadil/lithops
+        flags: k8s

.github/workflows/localhost-coverage.yml

@@ -0,0 +1,43 @@
+# This workflow will run a localhost backend test.
+name: Localhost Coverage
+on:
+  push:
+    branches: [ "add-code-coverage" ]
+    paths:
+      - 'lithops/**'
+  pull_request:
+    branches: [ "add-code-coverage" ]
+    paths:
+      - 'lithops/**'
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10"]
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install pytest
+        python -m pip install pytest-cov
+    - name: Install lithops
+      run: |
+        pip install .
+    - name: Test localhost with pytest
+      run: |        
+        coverage run --omit=./lithops/serverless/*,./lithops/standalone/*,./lithops/util/*,./lithops/libs/*,./lithops/localhost/v2/*,./lithops/utils.py,./lithops/tests/util_func/*,./lithops/scripts/**,./lithops/tests/test_template.py,./lithops/tests/tests_main.py -m pytest ./lithops/tests/github_coverage.py -k localhost
+        coverage xml
+    - name: Upload coverage to Codecov
+      uses: codecov/codecov-action@v4.0.1
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}
+        verbose: true
+        slug: acanadil/lithops
+        flags: localhost

README.md

@@ -16,6 +16,21 @@
   </a>
 </p>
 
+<p align="center">
+  <a href="https://codecov.io/gh/acanadil/lithops"> 
+    <img src="https://codecov.io/gh/acanadil/lithops/graph/badge.svg?token=S60HMH9YTH"/> 
+  </a>
+  <a aria-label="Localhost testing" href="https://github.com/acanadil/lithops/actions/workflows/localhost-coverage.yml"> 
+    <img src="https://github.com/acanadil/lithops/actions/workflows/localhost-coverage.yml/badge.svg"/> 
+  </a>
+  <a aria-label="AWS testing" href="https://github.com/acanadil/lithops/actions/workflows/aws-coverage.yml" > 
+    <img src="https://github.com/acanadil/lithops/actions/workflows/aws-coverage.yml/badge.svg"/> 
+  </a>
+  <a aria-label="K8s testing" href="https://github.com/acanadil/lithops/actions/workflows/k8s-coverage.yml"> 
+    <img src="https://github.com/acanadil/lithops/actions/workflows/k8s-coverage.yml/badge.svg"/> 
+  </a>
+</p>
+
 Lithops is a Python multi-cloud distributed computing framework. It allows you to run unmodified local python code at massive scale in the main
 serverless computing platforms. Lithops delivers the user’s code into the cloud without requiring knowledge of how it is deployed and run. Moreover, its multicloud-agnostic architecture ensures portability across cloud providers.
 

codecov.yaml

@@ -0,0 +1,14 @@
+coverage:
+  precision: 0
+  range: 50..75
+  round: up
+
+flags:
+  localhost:
+    carryforward: true
+
+  aws:
+    carryforward: true
+
+  k8s:
+    carryforward: true

lithops/tests/github_coverage.py

@@ -0,0 +1,41 @@
+#
+# (C) Copyright Cloudlab URV 2024
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from lithops.tests.tests_main import run_tests
+from lithops.scripts.cli import runtime, lithops_cli
+
+
+def test_github_localhost():
+    run_tests("all", None, "", None, None, None, False, False)
+
+
+def test_github_k8s():
+    runtime(['build', 'acanadilla/k8s_internal', '--file', './runtime/kubernetes/Dockerfile', '--backend', 'k8s'], standalone_mode=False)
+    run_tests("", None, "call_async", None, None, None, False, False)
+    run_tests("", None, "storage", None, None, None, False, False)
+    run_tests("", None, "map_reduce", None, None, None, False, False)
+    runtime(['build', 'acanadilla/test-bucket-k8s', '--file', './runtime/kubernetes/Dockerfile', '--backend', 'k8s'], standalone_mode=False)
+    runtime(['deploy', 'acanadilla/test-bucket-k8s', '--memory', 500], standalone_mode=False)
+    runtime(['list'], standalone_mode=False)
+    lithops_cli(['clean'], standalone_mode=False)
+
+
+def test_github_aws():
+    run_tests("all", None, "", None, None, None, False, False)
+    runtime(['build', 'acanadilla/test-bucket-s3', '--file', './runtime/aws_lambda/Dockerfile', '--backend', 'aws_lambda'], standalone_mode=False)
+    runtime(['deploy', 'acanadilla/test-bucket-s3', '--memory', 500], standalone_mode=False)
+    runtime(['list'], standalone_mode=False)
+    lithops_cli(['clean'], standalone_mode=False)