diff --git a/.github/workflows/aws-coverage.yml b/.github/workflows/aws-coverage.yml new file mode 100644 index 000000000..f89108db6 --- /dev/null +++ b/.github/workflows/aws-coverage.yml @@ -0,0 +1,57 @@ +# This workflow will run an aws backend test. +name: AWS coverage +on: + push: + branches: [ "add-code-coverage" ] + paths: + - 'lithops/serverless/**' + - '!lithops/serverless/backends/k8s/**' + - '.github/workflows/aws-coverage.yml' + pull_request: + branches: [ "add-code-coverage" ] + paths: + - 'lithops/serverless/**' + - '!lithops/serverless/backends/k8s/**' + - '.github/workflows/aws-coverage.yml' +jobs: + build_aws: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.10"] + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install pytest + python -m pip install pytest-cov + - name: Install lithops + run: | + pip install . + - name: Test aws with pytest + run: | + echo "lithops:" > .lithops_config + echo " backend: aws_lambda" >> .lithops_config + echo " storage: aws_s3" >> .lithops_config + echo "aws:" >> .lithops_config + echo " region: us-east-1" >> .lithops_config + echo " access_key_id: ${{ secrets.AWS_KEY_ID }}" >> .lithops_config + echo " secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}" >> .lithops_config + echo "aws_lambda:" >> .lithops_config + echo " execution_role: ${{ secrets.AWS_ROLE }}" >> .lithops_config + coverage run --omit=./lithops/standalone/*,./lithops/util/*,./lithops/libs/*,./lithops/localhost/*,./lithops/utils.py,./lithops/tests/util_func/*,./lithops/scripts/**,./lithops/tests/test_template.py,./lithops/tests/tests_main.py -m pytest ./lithops/tests/github_coverage.py -k aws + coverage xml + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v4.0.1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + verbose: true + slug: acanadil/lithops + flags: aws + diff --git a/.github/workflows/k8s-coverage.yml b/.github/workflows/k8s-coverage.yml new file mode 100644 index 000000000..26d356383 --- /dev/null +++ b/.github/workflows/k8s-coverage.yml @@ -0,0 +1,100 @@ +# This workflow will run a k8s backend. +name: K8s coverage +on: + push: + branches: [ "add-code-coverage" ] + paths: + - 'lithops/serverless/**' + - '!lithops/serverless/backends/aws_lambda/**' + - '.github/workflows/k8s-coverage.yml' + - 'lithops/storage/**' + pull_request: + branches: [ "add-code-coverage" ] + paths: + - 'lithops/serverless/**' + - '!lithops/serverless/backends/aws_lambda/**' + - '.github/workflows/k8s-coverage.yml' + - 'lithops/storage/**' +jobs: + build_k8s: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.10"] + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install pytest + python -m pip install pytest-cov + - name: Install lithops + run: | + pip install . + - name: Install docker + run: | + sudo apt-get update + sudo apt-get install ca-certificates curl + sudo install -m 0755 -d /etc/apt/keyrings + sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc + sudo chmod a+r /etc/apt/keyrings/docker.asc + echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \ + $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \ + sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + sudo apt-get update + sudo apt-get install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin + sudo docker run hello-world + sudo usermod -aG docker $USER + docker login -u ${{ secrets.DOCKER_USER }} -p ${{ secrets.DOCKER_TOKEN }} + - name: Install minikube + run: | + curl -LO https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64 + sudo install minikube-linux-amd64 /usr/local/bin/minikube + minikube start + - name: Install kubectl + run: | + curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" + sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl + - name: Wait until aws-coverage finishes + run: | + status="in_progress" + check="completed" + while [ "$check" != "$status" ]; do + response=$(curl -H "Accept: application/vnd.github.v3+json" "https://api.github.com/repos/$GITHUB_REPOSITORY/actions/runs?&head_sha=$GITHUB_SHA") + status=$(echo "$response" | jq -r '.workflow_runs[] | select(.path == ".github/workflows/aws-coverage.yml") | .status') + if [ "$status" == "" ]; then + status="completed" + fi + sleep 30 + done + - name: Test k8s with pytest + run: | + echo "lithops:" > .lithops_config + echo " backend: k8s" >> .lithops_config + echo " storage: aws_s3" >> .lithops_config + echo "k8s:" >> .lithops_config + echo " docker_server: docker.io" >> .lithops_config + echo " docker_user: ${{ secrets.DOCKER_USER }}" >> .lithops_config + echo " docker_password: ${{ secrets.DOCKER_TOKEN }}" >> .lithops_config + echo " runtime: ${{ secrets.DOCKER_RUNTIME }}" >> .lithops_config + echo " runtime_memory: 500" >> .lithops_config + echo " runtime_cpu: 1" >> .lithops_config + echo "aws:" >> .lithops_config + echo " region: us-east-1" >> .lithops_config + echo " access_key_id: ${{ secrets.AWS_KEY_ID }}" >> .lithops_config + echo " secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}" >> .lithops_config + coverage run --omit=./lithops/standalone/*,./lithops/util/*,./lithops/libs/*,./lithops/localhost/*,./lithops/utils.py,./lithops/tests/util_func/*,./lithops/scripts/**,./lithops/tests/test_template.py,./lithops/tests/tests_main.py -m pytest ./lithops/tests/github_coverage.py -k k8s + coverage xml + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v4.0.1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + verbose: true + slug: acanadil/lithops + flags: k8s diff --git a/.github/workflows/localhost-coverage.yml b/.github/workflows/localhost-coverage.yml new file mode 100644 index 000000000..59d55d0e9 --- /dev/null +++ b/.github/workflows/localhost-coverage.yml @@ -0,0 +1,44 @@ +# This workflow will run a localhost backend test. +name: Localhost Coverage +on: + push: + branches: [ "add-code-coverage" ] + paths: + - 'lithops/**' + - '.github/workflows/localhost-coverage.yml' + pull_request: + branches: [ "add-code-coverage" ] + paths: + - 'lithops/**' +jobs: + build: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.10"] + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install pytest + python -m pip install pytest-cov + - name: Install lithops + run: | + pip install . + - name: Test localhost with pytest + run: | + coverage run --omit=./lithops/serverless/*,./lithops/standalone/*,./lithops/util/*,./lithops/libs/*,./lithops/localhost/v2/*,./lithops/utils.py,./lithops/tests/util_func/*,./lithops/scripts/**,./lithops/tests/test_template.py,./lithops/tests/tests_main.py -m pytest ./lithops/tests/github_coverage.py -k localhost + coverage xml + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v4.0.1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + verbose: true + slug: acanadil/lithops + flags: localhost diff --git a/README.md b/README.md index 90394c358..bf1ea1c53 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,21 @@

+

+ + + + + + + + + + + + +

+ Lithops is a Python multi-cloud distributed computing framework. It allows you to run unmodified local python code at massive scale in the main serverless computing platforms. Lithops delivers the user’s code into the cloud without requiring knowledge of how it is deployed and run. Moreover, its multicloud-agnostic architecture ensures portability across cloud providers. diff --git a/codecov.yaml b/codecov.yaml new file mode 100644 index 000000000..16cfde1e3 --- /dev/null +++ b/codecov.yaml @@ -0,0 +1,14 @@ +coverage: + precision: 0 + range: 50..75 + round: up + +flags: + localhost: + carryforward: true + + aws: + carryforward: true + + k8s: + carryforward: true \ No newline at end of file diff --git a/lithops/tests/github_coverage.py b/lithops/tests/github_coverage.py new file mode 100644 index 000000000..1ab7df2d8 --- /dev/null +++ b/lithops/tests/github_coverage.py @@ -0,0 +1,41 @@ +# +# (C) Copyright Cloudlab URV 2024 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from lithops.tests.tests_main import run_tests +from lithops.scripts.cli import runtime, lithops_cli + + +def test_github_localhost(): + run_tests("all", None, "", None, None, None, False, False) + + +def test_github_k8s(): + runtime(['build', 'acanadilla/k8s_internal', '--file', './runtime/kubernetes/Dockerfile', '--backend', 'k8s'], standalone_mode=False) + run_tests("", None, "call_async", None, None, None, False, False) + run_tests("", None, "storage", None, None, None, False, False) + run_tests("", None, "map_reduce", None, None, None, False, False) + runtime(['build', 'acanadilla/test-bucket-k8s', '--file', './runtime/kubernetes/Dockerfile', '--backend', 'k8s'], standalone_mode=False) + runtime(['deploy', 'acanadilla/test-bucket-k8s', '--memory', 500], standalone_mode=False) + runtime(['list'], standalone_mode=False) + lithops_cli(['clean'], standalone_mode=False) + + +def test_github_aws(): + run_tests("all", None, "", None, None, None, False, False) + runtime(['build', 'acanadilla/test-bucket-s3', '--file', './runtime/aws_lambda/Dockerfile', '--backend', 'aws_lambda'], standalone_mode=False) + runtime(['deploy', 'acanadilla/test-bucket-s3', '--memory', 500], standalone_mode=False) + runtime(['list'], standalone_mode=False) + lithops_cli(['clean'], standalone_mode=False)