From 66df89e73711da51ab9f73ccf4672c5c18cc0b80 Mon Sep 17 00:00:00 2001 From: ykim-1 <126618609+ykim-1@users.noreply.github.com> Date: Tue, 6 Jun 2023 09:15:48 -0700 Subject: [PATCH 1/8] Add integration tests (#240) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## 📝 Description Adding integration tests - more details at:https://jira.linode.com/browse/TPT-1831 ## ✔️ How to Test **What are the steps to reproduce the issue or verify the changes?** **How do I run the relevant unit/integration tests?** ## 📷 Preview **If applicable, include a screenshot or code snippet of this change. Otherwise, please remove this section.** --------- Co-authored-by: Zhiwei Liang <121905282+zliang-akamai@users.noreply.github.com> --- .github/workflows/e2e-test-pr-command.yml | 19 + .github/workflows/e2e-test-pr.yml | 81 ++++ Makefile | 6 + requirements-dev.txt | 1 + requirements.txt | 2 +- test/integration/__init__.py | 0 test/integration/conftest.py | 218 +++++++++ test/integration/helpers.py | 115 +++++ test/integration/linode_client/__init__.py | 0 .../linode_client/test_linode_client.py | 406 ++++++++++++++++ test/integration/models/__init__.py | 0 test/integration/models/test_account.py | 99 ++++ test/integration/models/test_database.py | 445 ++++++++++++++++++ test/integration/models/test_domain.py | 60 +++ test/integration/models/test_firewall.py | 82 ++++ test/integration/models/test_image.py | 50 ++ test/integration/models/test_linode.py | 431 +++++++++++++++++ test/integration/models/test_lke.py | 137 ++++++ test/integration/models/test_longview.py | 45 ++ test/integration/models/test_networking.py | 12 + test/integration/models/test_nodebalancer.py | 119 +++++ test/integration/models/test_tag.py | 21 + test/integration/models/test_volume.py | 104 ++++ tox.ini | 2 +- 24 files changed, 2453 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/e2e-test-pr-command.yml create mode 100644 .github/workflows/e2e-test-pr.yml create mode 100644 test/integration/__init__.py create mode 100644 test/integration/conftest.py create mode 100644 test/integration/helpers.py create mode 100644 test/integration/linode_client/__init__.py create mode 100644 test/integration/linode_client/test_linode_client.py create mode 100644 test/integration/models/__init__.py create mode 100644 test/integration/models/test_account.py create mode 100644 test/integration/models/test_database.py create mode 100644 test/integration/models/test_domain.py create mode 100644 test/integration/models/test_firewall.py create mode 100644 test/integration/models/test_image.py create mode 100644 test/integration/models/test_linode.py create mode 100644 test/integration/models/test_lke.py create mode 100644 test/integration/models/test_longview.py create mode 100644 test/integration/models/test_networking.py create mode 100644 test/integration/models/test_nodebalancer.py create mode 100644 test/integration/models/test_tag.py create mode 100644 test/integration/models/test_volume.py diff --git a/.github/workflows/e2e-test-pr-command.yml b/.github/workflows/e2e-test-pr-command.yml new file mode 100644 index 000000000..3b52a695b --- /dev/null +++ b/.github/workflows/e2e-test-pr-command.yml @@ -0,0 +1,19 @@ +name: AccTest Command + +on: + issue_comment: + types: [created] + +jobs: + acctest-command: + runs-on: ubuntu-latest + if: ${{ github.event.issue.pull_request }} + steps: + - name: Slash Command Dispatch + uses: peter-evans/slash-command-dispatch@v1.2.0 + with: + token: ${{ secrets.GITHUB_TOKEN }} + issue-type: pull-request + commands: acctest + named-args: true + permission: write diff --git a/.github/workflows/e2e-test-pr.yml b/.github/workflows/e2e-test-pr.yml new file mode 100644 index 000000000..020874a66 --- /dev/null +++ b/.github/workflows/e2e-test-pr.yml @@ -0,0 +1,81 @@ +on: + pull_request: + repository_dispatch: + types: [acctest-command] + +name: PR E2E Tests + +jobs: + # Maintainer has commented /acctest on a pull request + integration-fork-ubuntu: + runs-on: ubuntu-latest + if: + github.event_name == 'repository_dispatch' && + github.event.client_payload.slash_command.sha != '' && + github.event.client_payload.pull_request.head.sha == github.event.client_payload.slash_command.sha + + steps: + - uses: actions-ecosystem/action-regex-match@v2 + id: validate-tests + with: + text: ${{ github.event.client_payload.slash_command.tests }} + regex: '[^a-z0-9-:.\/_]' # Tests validation + flags: gi + + # Check out merge commit + - name: Checkout PR + uses: actions/checkout@v3 + with: + ref: ${{ github.event.client_payload.slash_command.sha }} + + - name: Update system packages + run: sudo apt-get update -y + + - name: Install system deps + run: sudo apt-get install -y build-essential + + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.x' + + - name: Install Python deps + run: pip install -r requirements.txt -r requirements-dev.txt wheel boto3 + + - name: Install Python SDK + run: make install + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - run: make testint + if: ${{ steps.validate-tests.outputs.match == '' }} + env: + LINODE_CLI_TOKEN: ${{ secrets.LINODE_TOKEN }} + + - uses: actions/github-script@v5 + id: update-check-run + if: ${{ always() }} + env: + number: ${{ github.event.client_payload.pull_request.number }} + job: ${{ github.job }} + conclusion: ${{ job.status }} + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const { data: pull } = await github.rest.pulls.get({ + ...context.repo, + pull_number: process.env.number + }); + const ref = pull.head.sha; + const { data: checks } = await github.rest.checks.listForRef({ + ...context.repo, + ref + }); + const check = checks.check_runs.filter(c => c.name === process.env.job); + const { data: result } = await github.rest.checks.update({ + ...context.repo, + check_run_id: check[0].id, + status: 'completed', + conclusion: process.env.conclusion + }); + return result; \ No newline at end of file diff --git a/Makefile b/Makefile index e51e574bc..cf6c2431d 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,7 @@ PYTHON ?= python3 +INTEGRATION_TEST_PATH := + @PHONEY: clean clean: mkdir -p dist @@ -45,3 +47,7 @@ lint: autoflake --check linode_api4 test black --check --verbose linode_api4 test pylint linode_api4 + +@PHONEY: testint +testint: + python3 -m pytest test/integration/ diff --git a/requirements-dev.txt b/requirements-dev.txt index d0597c403..5e2d3165f 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -7,4 +7,5 @@ tox>=4.4.0 Sphinx>=6.0.0 sphinx-autobuild>=2021.3.14 sphinxcontrib-fulltoc>=1.2.0 +pytest>=7.3.1 httpretty>=1.1.4 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 9fe34db3d..3d78232b6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ httplib2 enum34 -requests \ No newline at end of file +requests diff --git a/test/integration/__init__.py b/test/integration/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/test/integration/conftest.py b/test/integration/conftest.py new file mode 100644 index 000000000..b3fa15fb2 --- /dev/null +++ b/test/integration/conftest.py @@ -0,0 +1,218 @@ +import os +import time + +import pytest + +from linode_api4.linode_client import LinodeClient, LongviewSubscription + +ENV_TOKEN_NAME = "LINODE_TOKEN" +RUN_LONG_TESTS = "RUN_LONG_TESTS" + + +def get_token(): + return os.environ.get(ENV_TOKEN_NAME, None) + + +def run_long_tests(): + return os.environ.get(RUN_LONG_TESTS, None) + + +@pytest.fixture(scope="session") +def create_linode(get_client): + client = get_client + available_regions = client.regions() + chosen_region = available_regions[0] + timestamp = str(int(time.time())) + label = "TestSDK-" + timestamp + + linode_instance, password = client.linode.instance_create( + "g5-standard-4", chosen_region, image="linode/debian9", label=label + ) + + yield linode_instance + + linode_instance.delete() + + +@pytest.fixture +def create_linode_for_pass_reset(get_client): + client = get_client + available_regions = client.regions() + chosen_region = available_regions[0] + timestamp = str(int(time.time())) + label = "TestSDK-" + timestamp + + linode_instance, password = client.linode.instance_create( + "g5-standard-4", chosen_region, image="linode/debian9", label=label + ) + + yield linode_instance, password + + linode_instance.delete() + + +@pytest.fixture(scope="session") +def ssh_key_gen(): + output = os.popen("ssh-keygen -q -t rsa -f ./sdk-sshkey -q -N ''") + + time.sleep(1) + + pub_file = open("./sdk-sshkey.pub", "r") + pub_key = pub_file.read().rstrip() + + priv_file = open("./sdk-sshkey", "r") + priv_key = priv_file.read().rstrip() + + yield pub_key, priv_key + + os.popen("rm ./sdk-sshkey*") + + +@pytest.fixture(scope="session") +def get_client(): + token = get_token() + client = LinodeClient(token) + return client + + +@pytest.fixture +def set_account_settings(get_client): + client = get_client + account_settings = client.account.settings() + account_settings._populated = True + account_settings.network_helper = True + + account_settings.save() + + +@pytest.fixture(scope="session") +def create_domain(get_client): + client = get_client + + timestamp = str(int(time.time())) + domain_addr = timestamp + "-example.com" + soa_email = "pathiel-test123@linode.com" + + domain = client.domain_create( + domain=domain_addr, soa_email=soa_email, tags=["test-tag"] + ) + + # Create a SRV record + domain.record_create( + "SRV", + target="rc_test", + priority=10, + weight=5, + port=80, + service="service_test", + ) + + yield domain + + domain.delete() + + +@pytest.fixture(scope="session") +def create_volume(get_client): + client = get_client + timestamp = str(int(time.time())) + label = "TestSDK-" + timestamp + + volume = client.volume_create(label=label, region="ap-west") + + yield volume + + volume.delete() + + +@pytest.fixture +def create_tag(get_client): + client = get_client + + timestamp = str(int(time.time())) + label = "TestSDK-" + timestamp + + tag = client.tag_create(label=label) + + yield tag + + tag.delete() + + +@pytest.fixture +def create_nodebalancer(get_client): + client = get_client + + timestamp = str(int(time.time())) + label = "TestSDK-" + timestamp + + nodebalancer = client.nodebalancer_create(region="us-east", label=label) + + yield nodebalancer + + nodebalancer.delete() + + +@pytest.fixture +def create_longview_client(get_client): + client = get_client + timestamp = str(int(time.time())) + label = "TestSDK-" + timestamp + longview_client = client.longview.client_create(label=label) + + yield longview_client + + longview_client.delete() + + +@pytest.fixture +def upload_sshkey(get_client, ssh_key_gen): + pub_key = ssh_key_gen[0] + client = get_client + key = client.profile.ssh_key_upload(pub_key, "IntTestSDK-sshkey") + + yield key + + key.delete() + + +@pytest.fixture +def create_ssh_keys_object_storage(get_client): + client = get_client + label = "TestSDK-obj-storage-key" + key = client.object_storage.keys_create(label) + + yield key + + key.delete() + + +@pytest.fixture(scope="session") +def create_firewall(get_client): + client = get_client + rules = { + "outbound": [], + "outbound_policy": "DROP", + "inbound": [], + "inbound_policy": "ACCEPT", + } + + firewall = client.networking.firewall_create( + "test-firewall", rules=rules, status="enabled" + ) + + yield firewall + + firewall.delete() + + +@pytest.fixture +def create_oauth_client(get_client): + client = get_client + oauth_client = client.account.oauth_client_create( + "test-oauth-client", "https://localhost/oauth/callback" + ) + + yield oauth_client + + oauth_client.delete() diff --git a/test/integration/helpers.py b/test/integration/helpers.py new file mode 100644 index 000000000..eee46f385 --- /dev/null +++ b/test/integration/helpers.py @@ -0,0 +1,115 @@ +import random +import time +from typing import Callable + +from linode_api4 import PaginatedList +from linode_api4.errors import ApiError +from linode_api4.linode_client import LinodeClient + + +def get_test_label(): + unique_timestamp = str(int(time.time()) + random.randint(0, 1000)) + label = "IntTestSDK_" + unique_timestamp + return label + + +def delete_instance_with_test_kw(paginated_list: PaginatedList): + for i in paginated_list: + try: + if hasattr(i, "label"): + label = getattr(i, "label") + if "IntTestSDK" in str(label): + i.delete() + elif "lke" in str(label): + iso_created_date = getattr(i, "created") + created_time = int( + time.mktime(iso_created_date.timetuple()) + ) + timestamp = int(time.time()) + if (timestamp - created_time) < 86400: + i.delete() + elif hasattr(i, "domain"): + domain = getattr(i, "domain") + if "IntTestSDK" in domain: + i.delete() + except AttributeError as e: + if "IntTestSDK" in str(i.__dict__): + i.delete() + + +def delete_all_test_instances(client: LinodeClient): + tags = client.tags() + linodes = client.linode.instances() + images = client.images() + volumes = client.volumes() + nodebalancers = client.nodebalancers() + domains = client.domains() + longview_clients = client.longview.clients() + clusters = client.lke.clusters() + firewalls = client.networking.firewalls() + + delete_instance_with_test_kw(tags) + delete_instance_with_test_kw(linodes) + delete_instance_with_test_kw(images) + delete_instance_with_test_kw(volumes) + delete_instance_with_test_kw(nodebalancers) + delete_instance_with_test_kw(domains) + delete_instance_with_test_kw(longview_clients) + delete_instance_with_test_kw(clusters) + delete_instance_with_test_kw(firewalls) + + +def wait_for_condition( + interval: int, timeout: int, condition: Callable, *args +) -> object: + start_time = time.time() + while True: + if condition(*args): + break + + if time.time() - start_time > timeout: + raise TimeoutError("Wait for condition timeout error") + + time.sleep(interval) + + +# Retry function to help in case of requests sending too quickly before instance is ready +def retry_sending_request(retries: int, condition: Callable, *args) -> object: + curr_t = 0 + while curr_t < retries: + try: + curr_t += 1 + res = condition(*args) + return res + except ApiError: + if curr_t >= retries: + raise ApiError + time.sleep(5) + + +def send_request_when_resource_available( + timeout: int, func: Callable, *args +) -> object: + start_time = time.time() + + while True: + try: + res = func(*args) + return res + except ApiError as e: + if ( + e.status == 400 + or e.status == 500 + or "Please try again later" in str(e.__dict__) + ): + if time.time() - start_time > timeout: + raise TimeoutError( + "Timeout Error: resource is not available in" + + timeout + + "seconds" + ) + time.sleep(10) + else: + raise e + + return res diff --git a/test/integration/linode_client/__init__.py b/test/integration/linode_client/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/test/integration/linode_client/test_linode_client.py b/test/integration/linode_client/test_linode_client.py new file mode 100644 index 000000000..7549ae89a --- /dev/null +++ b/test/integration/linode_client/test_linode_client.py @@ -0,0 +1,406 @@ +import re +import time +from test.integration.helpers import get_test_label + +import pytest + +from linode_api4 import ApiError, LinodeClient +from linode_api4.objects import ObjectStorageKeys + + +@pytest.fixture(scope="session", autouse=True) +def setup_client_and_linode(get_client): + client = get_client + available_regions = client.regions() + chosen_region = available_regions[0] + label = get_test_label() + + linode_instance, password = client.linode.instance_create( + "g5-standard-4", chosen_region, image="linode/debian9", label=label + ) + + yield client, linode_instance + + linode_instance.delete() + + +def test_get_account(setup_client_and_linode): + client = setup_client_and_linode[0] + account = client.account() + + assert re.search("^$|[a-zA-Z]+", account.first_name) + assert re.search("^$|[a-zA-Z]+", account.last_name) + assert re.search( + "^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$", account.email + ) + assert re.search( + "^(\+\d{1,2}\s)?\(?\d{3}\)?[\s.-]?\d{3}[\s.-]?\d{4}$", account.phone + ) + assert re.search("^$|[a-zA-Z0-9]+", account.address_1) + assert re.search("^$|[a-zA-Z0-9]+", account.address_2) + assert re.search("^$|[a-zA-Z]+", account.city) + assert re.search("^$|[a-zA-Z]+", account.state) + assert re.search("^$|[a-zA-Z]+", account.country) + assert re.search("^$|[a-zA-Z0-9]+", account.zip) + if account.tax_id: + assert re.search("^$|[0-9]+", account.tax_id) + + +def test_fails_to_create_domain_without_soa_email(setup_client_and_linode): + client = setup_client_and_linode[0] + + timestamp = str(int(time.time())) + domain_addr = timestamp + "example.com" + try: + domain = client.domain_create(domain=domain_addr) + except ApiError as e: + assert e.status == 400 + + +def test_get_domains(get_client, create_domain): + client = get_client + domain = create_domain + domain_dict = client.domains() + + dom_list = [i.domain for i in domain_dict] + + assert domain.domain in dom_list + + +def test_image_create(setup_client_and_linode): + client = setup_client_and_linode[0] + linode = setup_client_and_linode[1] + + label = get_test_label() + description = "Test description" + disk_id = linode.disks.first().id + + image = client.image_create( + disk=disk_id, label=label, description=description + ) + + assert image.label == label + assert image.description == description + + +def test_fails_to_create_image_with_non_existing_disk_id( + setup_client_and_linode, +): + client = setup_client_and_linode[0] + + label = get_test_label() + description = "Test description" + disk_id = 111111 + + try: + image_page = client.image_create( + disk=disk_id, label=label, description=description + ) + except ApiError as e: + assert "Not found" in str(e.json) + assert e.status == 404 + + +def test_fails_to_delete_predefined_images(setup_client_and_linode): + client = setup_client_and_linode[0] + + images = client.images() + + try: + # new images go on top of the list thus choose last image + images.last().delete() + except ApiError as e: + assert "Unauthorized" in str(e.json) + assert e.status == 403 + + +def test_get_volume(get_client, create_volume): + client = get_client + label = create_volume.label + + volume_dict = client.volumes() + + volume_label_list = [i.label for i in volume_dict] + + assert label in volume_label_list + + +def test_get_tag(get_client, create_tag): + client = get_client + label = create_tag.label + + tags = client.tags() + + tag_label_list = [i.label for i in tags] + + assert label in tag_label_list + + +def test_create_tag_with_id( + setup_client_and_linode, create_nodebalancer, create_domain, create_volume +): + client = setup_client_and_linode[0] + linode = setup_client_and_linode[1] + nodebalancer = create_nodebalancer + domain = create_domain + volume = create_volume + + label = get_test_label() + + tag = client.tag_create( + label=label, + instances=[linode.id, linode], + nodebalancers=[nodebalancer.id, nodebalancer], + domains=[domain.id, domain], + volumes=[volume.id, volume], + ) + + # Get tags after creation + tags = client.tags() + + tag_label_list = [i.label for i in tags] + + tag.delete() + + assert label in tag_label_list + + +def test_create_tag_with_entities( + setup_client_and_linode, create_nodebalancer, create_domain, create_volume +): + client = setup_client_and_linode[0] + linode = setup_client_and_linode[1] + nodebalancer = create_nodebalancer + domain = create_domain + volume = create_volume + + label = get_test_label() + + tag = client.tag_create( + label, entities=[linode, domain, nodebalancer, volume] + ) + + # Get tags after creation + tags = client.tags() + + tag_label_list = [i.label for i in tags] + + tag.delete() + + assert label in tag_label_list + + +# AccountGroupTests +def test_get_account_settings(get_client): + client = get_client + account_settings = client.account.settings() + + assert account_settings._populated == True + assert re.search( + "'network_helper':True|False", str(account_settings._raw_json) + ) + + +# TODO: Account invoice and payment test cases need to be added + + +# LinodeGroupTests +def test_create_linode_instance_without_image(get_client): + client = get_client + available_regions = client.regions() + chosen_region = available_regions[0] + label = get_test_label() + + linode_instance = client.linode.instance_create( + "g5-standard-4", chosen_region, label=label + ) + + assert linode_instance.label == label + assert linode_instance.image is None + + res = linode_instance.delete() + + assert res + + +def test_create_linode_instance_with_image(setup_client_and_linode): + linode = setup_client_and_linode[1] + + assert re.search("linode/debian9", str(linode.image)) + + +# LongviewGroupTests +def test_get_longview_clients(get_client, create_longview_client): + client = get_client + + longview_client = client.longview.clients() + + client_labels = [i.label for i in longview_client] + + assert create_longview_client.label in client_labels + + +def test_client_create_with_label(get_client): + client = get_client + label = get_test_label() + longview_client = client.longview.client_create(label=label) + + assert label == longview_client.label + + time.sleep(5) + + res = longview_client.delete() + + assert res + + +# TODO: Subscription related test cases need to be added, currently returns a 404 +# def test_get_subscriptions(): + + +# LKEGroupTest + + +def test_kube_version(get_client): + client = get_client + lke_version = client.lke.versions() + + assert re.search("[0-9].[0-9]+", lke_version.first().id) + + +def test_cluster_create_with_api_objects(get_client): + client = get_client + node_type = client.linode.types()[1] # g6-standard-1 + version = client.lke.versions()[0] + region = client.regions().first() + node_pools = client.lke.node_pool(node_type, 3) + label = get_test_label() + "-cluster" + + cluster = client.lke.cluster_create(region, label, node_pools, version) + + assert cluster.region.id == region.id + assert cluster.k8s_version.id == version.id + + res = cluster.delete() + + assert res + + +def test_fails_to_create_cluster_with_invalid_version(get_client): + invalid_version = "a.12" + client = get_client + + try: + cluster = client.lke.cluster_create( + "ap-west", + "example-cluster", + {"type": "g6-standard-1", "count": 3}, + invalid_version, + ) + except ApiError as e: + assert "not valid" in str(e.json) + assert e.status == 400 + + +# ProfileGroupTest + + +def test_get_sshkeys(get_client, upload_sshkey): + client = get_client + + ssh_keys = client.profile.ssh_keys() + + ssh_labels = [i.label for i in ssh_keys] + + assert upload_sshkey.label in ssh_labels + + +def test_ssh_key_create(upload_sshkey, ssh_key_gen): + pub_key = ssh_key_gen[0] + key = upload_sshkey + + assert pub_key == key._raw_json["ssh_key"] + + +# ObjectStorageGroupTests + + +def test_get_object_storage_clusters(get_client): + client = get_client + + clusters = client.object_storage.clusters() + + assert "us-east" in clusters[0].id + assert "us-east" in clusters[0].region.id + + +def test_get_keys(get_client, create_ssh_keys_object_storage): + client = get_client + key = create_ssh_keys_object_storage + + keys = client.object_storage.keys() + key_labels = [i.label for i in keys] + + assert key.label in key_labels + + +def test_keys_create(get_client, create_ssh_keys_object_storage): + key = create_ssh_keys_object_storage + + assert type(key) == type(ObjectStorageKeys(client=get_client, id="123")) + + +# NetworkingGroupTests + +# TODO:: creating vlans +# def test_get_vlans(): + + +@pytest.fixture +def create_firewall_with_inbound_outbound_rules(get_client): + client = get_client + label = get_test_label() + "-firewall" + rules = { + "outbound": [ + { + "ports": "22", + "protocol": "TCP", + "addresses": {"ipv4": ["198.0.0.2/32"]}, + "action": "ACCEPT", + "label": "accept-inbound-SSH", + } + ], + "outbound_policy": "DROP", + "inbound": [ + { + "ports": "22", + "protocol": "TCP", + "addresses": {"ipv4": ["198.0.0.2/32"]}, + "action": "ACCEPT", + "label": "accept-inbound-SSH", + } + ], + "inbound_policy": "ACCEPT", + } + + firewall = client.networking.firewall_create( + label, rules=rules, status="enabled" + ) + + yield firewall + + firewall.delete() + + +def test_get_firewalls_with_inbound_outbound_rules( + get_client, create_firewall_with_inbound_outbound_rules +): + client = get_client + firewalls = client.networking.firewalls() + firewall = create_firewall_with_inbound_outbound_rules + + firewall_labels = [i.label for i in firewalls] + + assert firewall.label in firewall_labels + assert firewall.rules.inbound_policy == "ACCEPT" + assert firewall.rules.outbound_policy == "DROP" diff --git a/test/integration/models/__init__.py b/test/integration/models/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/test/integration/models/test_account.py b/test/integration/models/test_account.py new file mode 100644 index 000000000..46542545d --- /dev/null +++ b/test/integration/models/test_account.py @@ -0,0 +1,99 @@ +import time +from test.integration.helpers import get_test_label + +from linode_api4.objects import ( + Account, + AccountSettings, + Event, + Login, + OAuthClient, + User, +) + + +def test_get_account(get_client): + client = get_client + account = client.account() + account_id = account.id + account_get = client.load(Account, account_id) + + assert account_get.first_name == account.first_name + assert account_get.last_name == account.last_name + assert account_get.email == account.email + assert account_get.phone == account.phone + assert account_get.address_1 == account.address_1 + assert account_get.address_2 == account.address_2 + assert account_get.city == account.city + assert account_get.state == account.state + assert account_get.country == account.country + assert account_get.zip == account.zip + assert account_get.tax_id == account.tax_id + + +def test_get_login(get_client): + client = get_client + login = client.load(Login(client, "", {}), "") + + updated_time = int(time.mktime(getattr(login, "_last_updated").timetuple())) + + login_updated = int(time.time()) - updated_time + + assert "username" in str(login._raw_json) + assert "ip" in str(login._raw_json) + assert "datetime" in str(login._raw_json) + assert "status" in str(login._raw_json) + assert login_updated < 15 + + +def test_get_account_settings(get_client): + client = get_client + account_settings = client.load(AccountSettings(client, ""), "") + + assert "managed" in str(account_settings._raw_json) + assert "network_helper" in str(account_settings._raw_json) + assert "longview_subscription" in str(account_settings._raw_json) + assert "backups_enabled" in str(account_settings._raw_json) + assert "object_storage" in str(account_settings._raw_json) + + +def test_latest_get_event(get_client): + client = get_client + + available_regions = client.regions() + chosen_region = available_regions[0] + label = get_test_label() + + linode, password = client.linode.instance_create( + "g5-standard-4", chosen_region, image="linode/debian9", label=label + ) + + events = client.load(Event, "") + + latest_event = events._raw_json.get("data")[0] + + linode.delete() + + assert label in latest_event["entity"]["label"] + + +def test_get_oathclient(get_client, create_oauth_client): + client = get_client + + oauth_client = client.load(OAuthClient, create_oauth_client.id) + + assert "test-oauth-client" == oauth_client.label + assert "https://localhost/oauth/callback" == oauth_client.redirect_uri + + +def test_get_user(get_client): + client = get_client + + events = client.load(Event, "") + + username = events._raw_json.get("data")[0]["username"] + + user = client.load(User, username) + + assert username == user.username + assert "email" in user._raw_json + assert "email" in user._raw_json diff --git a/test/integration/models/test_database.py b/test/integration/models/test_database.py new file mode 100644 index 000000000..974c5c923 --- /dev/null +++ b/test/integration/models/test_database.py @@ -0,0 +1,445 @@ +import re +import time +from test.integration.helpers import ( + get_test_label, + send_request_when_resource_available, + wait_for_condition, +) + +import pytest + +from linode_api4 import LinodeClient +from linode_api4.objects import MySQLDatabase, PostgreSQLDatabase + + +# Test Helpers +def get_db_engine_id(client: LinodeClient, engine: str): + engines = client.database.engines() + engine_id = "" + for e in engines: + if e.engine == engine: + engine_id = e.id + + return str(engine_id) + + +def get_sql_db_status(client: LinodeClient, db_id, status: str): + db = client.load(MySQLDatabase, db_id) + return db.status == status + + +def get_postgres_db_status(client: LinodeClient, db_id, status: str): + db = client.load(PostgreSQLDatabase, db_id) + return db.status == status + + +@pytest.fixture(scope="session") +def test_create_sql_db(get_client): + client = get_client + label = get_test_label() + "-sqldb" + region = "us-east" + engine_id = get_db_engine_id(client, "mysql") + dbtype = "g6-standard-1" + + db = client.database.mysql_create( + label=label, + region=region, + engine=engine_id, + ltype=dbtype, + cluster_size=None, + ) + + def get_db_status(): + return db.status == "active" + + # TAKES 15-30 MINUTES TO FULLY PROVISION DB + wait_for_condition(60, 2000, get_db_status) + + yield db + + send_request_when_resource_available(300, db.delete) + + +@pytest.fixture(scope="session") +def test_create_postgres_db(get_client): + client = get_client + label = get_test_label() + "-postgresqldb" + region = "us-east" + engine_id = get_db_engine_id(client, "postgresql") + dbtype = "g6-standard-1" + + db = client.database.postgresql_create( + label=label, + region=region, + engine=engine_id, + ltype=dbtype, + cluster_size=None, + ) + + def get_db_status(): + return db.status == "active" + + # TAKES 15-30 MINUTES TO FULLY PROVISION DB + wait_for_condition(60, 2000, get_db_status) + + yield db + + send_request_when_resource_available(300, db.delete) + + +# ------- SQL DB Test cases ------- +def test_get_types(get_client): + client = get_client + types = client.database.types() + + assert (types[0].type_class, "nanode") + assert (types[0].id, "g6-nanode-1") + assert (types[0].engines.mongodb[0].price.monthly, 15) + + +def test_get_engines(get_client): + client = get_client + engines = client.database.engines() + + for e in engines: + assert e.engine in ["mysql", "postgresql"] + assert re.search("[0-9]+.[0-9]+", e.version) + assert e.id == e.engine + "/" + e.version + + +def test_database_instance(get_client, test_create_sql_db): + dbs = get_client.database.mysql_instances() + + assert str(test_create_sql_db.id) in str(dbs.lists) + + +# ------- POSTGRESQL DB Test cases ------- +def test_get_sql_db_instance(get_client, test_create_sql_db): + dbs = get_client.database.mysql_instances() + database = "" + for db in dbs: + if db.id == test_create_sql_db.id: + database = db + + assert str(test_create_sql_db.id) == str(database.id) + assert str(test_create_sql_db.label) == str(database.label) + assert database.cluster_size == 1 + assert database.engine == "mysql" + assert "-mysql-primary.servers.linodedb.net" in database.hosts.primary + + +def test_update_sql_db(get_client, test_create_sql_db): + db = get_client.load(MySQLDatabase, test_create_sql_db.id) + + new_allow_list = ["192.168.0.1/32"] + label = get_test_label() + "updatedSQLDB" + + db.allow_list = new_allow_list + db.updates.day_of_week = 2 + db.label = label + + res = db.save() + + database = get_client.load(MySQLDatabase, test_create_sql_db.id) + + wait_for_condition( + 30, 300, get_sql_db_status, get_client, test_create_sql_db.id, "active" + ) + + assert res + assert database.allow_list == new_allow_list + assert database.label == label + assert database.updates.day_of_week == 2 + + +def test_create_sql_backup(get_client, test_create_sql_db): + db = get_client.load(MySQLDatabase, test_create_sql_db.id) + label = "database_backup_test" + + wait_for_condition( + 30, 300, get_sql_db_status, get_client, test_create_sql_db.id, "active" + ) + + db.backup_create(label=label, target="secondary") + + wait_for_condition( + 10, + 300, + get_sql_db_status, + get_client, + test_create_sql_db.id, + "backing_up", + ) + + assert db.status == "backing_up" + + # list backup and most recently created one is first element of the array + wait_for_condition( + 30, 600, get_sql_db_status, get_client, test_create_sql_db.id, "active" + ) + + backup = db.backups[0] + + assert backup.label == label + assert backup.database_id == test_create_sql_db.id + + assert db.status == "active" + + backup.delete() + + +def test_sql_backup_restore(get_client, test_create_sql_db): + db = get_client.load(MySQLDatabase, test_create_sql_db.id) + try: + backup = db.backups[0] + except IndexError as e: + pytest.skip( + "Skipping this test. Reason: Couldn't find db backup instance" + ) + + backup.restore() + + wait_for_condition( + 10, + 300, + get_sql_db_status, + get_client, + test_create_sql_db.id, + "restoring", + ) + + assert db.status == "restoring" + + wait_for_condition( + 30, 1000, get_sql_db_status, get_client, test_create_sql_db.id, "active" + ) + + assert db.status == "active" + + +def test_get_sql_ssl(get_client, test_create_sql_db): + db = get_client.load(MySQLDatabase, test_create_sql_db.id) + + assert "ca_certificate" in str(db.ssl) + + +def test_sql_patch(get_client, test_create_sql_db): + db = get_client.load(MySQLDatabase, test_create_sql_db.id) + + db.patch() + + wait_for_condition( + 10, + 300, + get_sql_db_status, + get_client, + test_create_sql_db.id, + "updating", + ) + + assert db.status == "updating" + + wait_for_condition( + 30, 1000, get_sql_db_status, get_client, test_create_sql_db.id, "active" + ) + + assert db.status == "active" + + +def test_get_sql_credentials(get_client, test_create_sql_db): + db = get_client.load(MySQLDatabase, test_create_sql_db.id) + + assert db.credentials.username == "linroot" + assert db.credentials.password + + +def test_reset_sql_credentials(get_client, test_create_sql_db): + db = get_client.load(MySQLDatabase, test_create_sql_db.id) + + old_pass = str(db.credentials.password) + + print(old_pass) + db.credentials_reset() + + time.sleep(5) + + assert db.credentials.username == "linroot" + assert db.credentials.password != old_pass + + +# ------- POSTGRESQL DB Test cases ------- +def test_get_postgres_db_instance(get_client, test_create_postgres_db): + dbs = get_client.database.postgresql_instances() + + for db in dbs: + if db.id == test_create_postgres_db.id: + database = db + + assert str(test_create_postgres_db.id) == str(database.id) + assert str(test_create_postgres_db.label) == str(database.label) + assert database.cluster_size == 1 + assert database.engine == "postgresql" + assert "pgsql-primary.servers.linodedb.net" in database.hosts.primary + + +def test_update_postgres_db(get_client, test_create_postgres_db): + db = get_client.load(PostgreSQLDatabase, test_create_postgres_db.id) + + new_allow_list = ["192.168.0.1/32"] + label = get_test_label() + "updatedPostgresDB" + + db.allow_list = new_allow_list + db.updates.day_of_week = 2 + db.label = label + + res = db.save() + + database = get_client.load(PostgreSQLDatabase, test_create_postgres_db.id) + + wait_for_condition( + 30, + 1000, + get_postgres_db_status, + get_client, + test_create_postgres_db.id, + "active", + ) + + assert res + assert database.allow_list == new_allow_list + assert database.label == label + assert database.updates.day_of_week == 2 + + +def test_create_postgres_backup(get_client, test_create_postgres_db): + pytest.skip( + "Failing due to '400: The backup snapshot request failed, please contact support.'" + ) + db = get_client.load(PostgreSQLDatabase, test_create_postgres_db.id) + label = "database_backup_test" + + wait_for_condition( + 30, + 1000, + get_postgres_db_status, + get_client, + test_create_postgres_db.id, + "active", + ) + + db.backup_create(label=label, target="secondary") + + # list backup and most recently created one is first element of the array + wait_for_condition( + 10, + 300, + get_sql_db_status, + get_client, + test_create_postgres_db.id, + "backing_up", + ) + + assert db.status == "backing_up" + + # list backup and most recently created one is first element of the array + wait_for_condition( + 30, + 600, + get_sql_db_status, + get_client, + test_create_postgres_db.id, + "active", + ) + + # list backup and most recently created one is first element of the array + backup = db.backups[0] + + assert backup.label == label + assert backup.database_id == test_create_postgres_db.id + + +def test_postgres_backup_restore(get_client, test_create_postgres_db): + db = get_client.load(PostgreSQLDatabase, test_create_postgres_db.id) + + try: + backup = db.backups[0] + except IndexError as e: + pytest.skip( + "Skipping this test. Reason: Couldn't find db backup instance" + ) + + backup.restore() + + wait_for_condition( + 30, + 1000, + get_postgres_db_status, + get_client, + test_create_postgres_db.id, + "restoring", + ) + + wait_for_condition( + 30, + 1000, + get_postgres_db_status, + get_client, + test_create_postgres_db.id, + "active", + ) + + assert db.status == "active" + + +def test_get_postgres_ssl(get_client, test_create_postgres_db): + db = get_client.load(PostgreSQLDatabase, test_create_postgres_db.id) + + assert "ca_certificate" in str(db.ssl) + + +def test_postgres_patch(get_client, test_create_postgres_db): + db = get_client.load(PostgreSQLDatabase, test_create_postgres_db.id) + + db.patch() + + wait_for_condition( + 10, + 300, + get_postgres_db_status, + get_client, + test_create_postgres_db.id, + "updating", + ) + + assert db.status == "updating" + + wait_for_condition( + 30, + 600, + get_postgres_db_status, + get_client, + test_create_postgres_db.id, + "active", + ) + + assert db.status == "active" + + +def test_get_postgres_credentials(get_client, test_create_postgres_db): + db = get_client.load(PostgreSQLDatabase, test_create_postgres_db.id) + + assert db.credentials.username == "linpostgres" + assert db.credentials.password + + +def test_reset_postgres_credentials(get_client, test_create_postgres_db): + db = get_client.load(PostgreSQLDatabase, test_create_postgres_db.id) + + old_pass = str(db.credentials.password) + + db.credentials_reset() + + time.sleep(5) + + assert db.credentials.username == "linpostgres" + assert db.credentials.password != old_pass diff --git a/test/integration/models/test_domain.py b/test/integration/models/test_domain.py new file mode 100644 index 000000000..2185a53d1 --- /dev/null +++ b/test/integration/models/test_domain.py @@ -0,0 +1,60 @@ +import re +import time +from test.integration.helpers import wait_for_condition + +import pytest + +from linode_api4.objects import Domain, DomainRecord + + +def test_get_domain_record(get_client, create_domain): + dr = DomainRecord( + get_client, create_domain.records.first().id, create_domain.id + ) + + assert dr.id == create_domain.records.first().id + + +def test_save_null_values_excluded(get_client, create_domain): + domain = get_client.load(Domain, create_domain.id) + + domain.type = "master" + domain.master_ips = ["127.0.0.1"] + res = domain.save() + + assert res + + +def test_zone_file_view(get_client, create_domain): + domain = get_client.load(Domain, create_domain.id) + + def get_zone_file_view(): + res = domain.zone_file_view() + return res != [] + + wait_for_condition(10, 100, get_zone_file_view) + + assert domain.domain in str(domain.zone_file_view()) + assert re.search("ns[0-9].linode.com", str(domain.zone_file_view())) + + +def test_clone(get_client, create_domain): + domain = get_client.load(Domain, create_domain.id) + timestamp = str(int(time.time())) + dom = "example.clone-" + timestamp + "-IntTestSDK.org" + domain.clone(dom) + + ds = get_client.domains() + + time.sleep(1) + + domains = [i.domain for i in ds] + + assert dom in domains + + +def test_import(get_client, create_domain): + pytest.skip( + 'Currently failing with message: linode_api4.errors.ApiError: 400: An unknown error occured. Please open a ticket for further assistance. Command: domain_import(domain, "google.ca")' + ) + domain = get_client.load(Domain, create_domain.id) diff --git a/test/integration/models/test_firewall.py b/test/integration/models/test_firewall.py new file mode 100644 index 000000000..6f0543516 --- /dev/null +++ b/test/integration/models/test_firewall.py @@ -0,0 +1,82 @@ +import time + +import pytest + +from linode_api4.objects import Firewall, FirewallDevice + + +@pytest.fixture(scope="session") +def create_linode_fw(get_client): + client = get_client + available_regions = client.regions() + chosen_region = available_regions[0] + label = "linode_instance_fw_device" + + linode_instance, password = client.linode.instance_create( + "g5-standard-4", chosen_region, image="linode/debian9", label=label + ) + + yield linode_instance + + linode_instance.delete() + + +def test_get_firewall_rules(get_client, create_firewall): + firewall = get_client.load(Firewall, create_firewall.id) + rules = firewall.rules + + assert rules.inbound_policy in ["ACCEPT", "DROP"] + assert rules.outbound_policy in ["ACCEPT", "DROP"] + + +def test_update_firewall_rules(get_client, create_firewall): + firewall = get_client.load(Firewall, create_firewall.id) + new_rules = { + "inbound": [ + { + "action": "ACCEPT", + "addresses": { + "ipv4": ["0.0.0.0/0"], + "ipv6": ["ff00::/8"], + }, + "description": "A really cool firewall rule.", + "label": "really-cool-firewall-rule", + "ports": "80", + "protocol": "TCP", + } + ], + "inbound_policy": "ACCEPT", + "outbound": [], + "outbound_policy": "DROP", + } + + firewall.update_rules(new_rules) + + time.sleep(1) + + firewall = get_client.load(Firewall, create_firewall.id) + + assert firewall.rules.inbound_policy == "ACCEPT" + assert firewall.rules.outbound_policy == "DROP" + + +def test_get_devices(get_client, create_linode_fw, create_firewall): + linode = create_linode_fw + + create_firewall.device_create(int(linode.id)) + + firewall = get_client.load(Firewall, create_firewall.id) + + assert len(firewall.devices) > 0 + + +def test_get_device(get_client, create_firewall, create_linode_fw): + firewall = create_firewall + + firewall_device = get_client.load( + FirewallDevice, firewall.devices.first().id, firewall.id + ) + + assert firewall_device.entity.label == "linode_instance_fw_device" + assert firewall_device.entity.type == "linode" + assert "/v4/linode/instances/" in firewall_device.entity.url diff --git a/test/integration/models/test_image.py b/test/integration/models/test_image.py new file mode 100644 index 000000000..6cd97d468 --- /dev/null +++ b/test/integration/models/test_image.py @@ -0,0 +1,50 @@ +from io import BytesIO +from test.integration.helpers import ( + delete_instance_with_test_kw, + get_test_label, +) + +import pytest + +from linode_api4.objects import Image + + +@pytest.fixture(scope="session") +def image_upload(get_client): + label = get_test_label() + "_image" + + get_client.image_create_upload( + label, "us-east", "integration test image upload" + ) + + image = get_client.images()[0] + + yield image + + image.delete() + images = get_client.images() + delete_instance_with_test_kw(images) + + +def test_get_image(get_client, image_upload): + image = get_client.load(Image, image_upload.id) + + assert image.label == image_upload.label + + +def test_image_create_upload(get_client): + test_image_content = ( + b"\x1F\x8B\x08\x08\xBD\x5C\x91\x60\x00\x03\x74\x65\x73\x74\x2E\x69" + b"\x6D\x67\x00\x03\x00\x00\x00\x00\x00\x00\x00\x00\x00" + ) + + label = get_test_label() + "_image" + image = get_client.image_upload( + label, + "us-east", + BytesIO(test_image_content), + description="integration test image upload", + ) + + assert image.label == label + assert image.description == "integration test image upload" diff --git a/test/integration/models/test_linode.py b/test/integration/models/test_linode.py new file mode 100644 index 000000000..a756ab944 --- /dev/null +++ b/test/integration/models/test_linode.py @@ -0,0 +1,431 @@ +import time +from test.integration.helpers import ( + get_test_label, + retry_sending_request, + wait_for_condition, +) + +import pytest + +from linode_api4.errors import ApiError +from linode_api4.objects import Config, Disk, Image, Instance, Type + + +@pytest.fixture(scope="session") +def create_linode_with_volume_firewall(get_client): + client = get_client + available_regions = client.regions() + chosen_region = available_regions[0] + label = get_test_label() + + rules = { + "outbound": [], + "outbound_policy": "DROP", + "inbound": [], + "inbound_policy": "ACCEPT", + } + + linode_instance, password = client.linode.instance_create( + "g5-standard-4", + chosen_region, + image="linode/debian9", + label=label + "_modlinode", + ) + + volume = client.volume_create( + label=label + "_volume", + region=linode_instance.region.id, + linode=linode_instance.id, + ) + + firewall = client.networking.firewall_create( + label=label + "_firewall", rules=rules, status="enabled" + ) + + firewall.device_create(int(linode_instance.id)) + + yield linode_instance + + firewall.delete() + + linode_instance.delete() + + volume.detach() + # wait for volume detach, can't currently get the attach/unattached status via SDK + time.sleep(30) + + volume.delete() + + +@pytest.fixture +def create_linode_for_long_running_tests(get_client): + client = get_client + available_regions = client.regions() + chosen_region = available_regions[0] + label = get_test_label() + + linode_instance, password = client.linode.instance_create( + "g5-standard-4", + chosen_region, + image="linode/debian9", + label=label + "_long_tests", + ) + + yield linode_instance + + linode_instance.delete() + + +# Test helper +def get_status(linode: Instance, status: str): + return linode.status == status + + +def test_get_linode(get_client, create_linode_with_volume_firewall): + linode = get_client.load(Instance, create_linode_with_volume_firewall.id) + + assert linode.label == create_linode_with_volume_firewall.label + assert linode.id == create_linode_with_volume_firewall.id + + +def test_linode_transfer(get_client, create_linode_with_volume_firewall): + linode = get_client.load(Instance, create_linode_with_volume_firewall.id) + + transfer = linode.transfer + + assert "used" in str(transfer) + assert "quota" in str(transfer) + assert "billable" in str(transfer) + + +def test_linode_rebuild(get_client): + client = get_client + available_regions = client.regions() + chosen_region = available_regions[0] + label = get_test_label() + "_rebuild" + + linode, password = client.linode.instance_create( + "g5-standard-4", chosen_region, image="linode/debian9", label=label + ) + + wait_for_condition(10, 100, get_status, linode, "running") + + retry_sending_request(3, linode.rebuild, "linode/debian9") + + wait_for_condition(10, 100, get_status, linode, "rebuilding") + + assert linode.status == "rebuilding" + assert linode.image.id == "linode/debian9" + + wait_for_condition(10, 300, get_status, linode, "running") + + assert linode.status == "running" + + linode.delete() + + +def test_linode_available_backups(create_linode): + linode = create_linode + + enable_backup = linode.enable_backups() + backups = linode.backups + + assert enable_backup + assert "enabled" in str(backups) + assert "available" in str(backups) + assert "schedule" in str(backups) + assert "last_successful" in str(backups) + + +def test_update_linode(create_linode): + linode = create_linode + new_label = get_test_label() + "_updated" + linode.label = new_label + linode.group = "new_group" + updated = linode.save() + + assert updated + assert linode.label == new_label + + +def test_delete_linode(get_client): + client = get_client + available_regions = client.regions() + chosen_region = available_regions[0] + label = get_test_label() + + linode_instance, password = client.linode.instance_create( + "g5-standard-4", + chosen_region, + image="linode/debian9", + label=label + "_linode", + ) + + linode_instance.delete() + + +def test_linode_reboot(create_linode): + linode = create_linode + + wait_for_condition(3, 100, get_status, linode, "running") + + retry_sending_request(3, linode.reboot) + + wait_for_condition(3, 100, get_status, linode, "rebooting") + assert linode.status == "rebooting" + + wait_for_condition(3, 100, get_status, linode, "running") + assert linode.status == "running" + + +def test_linode_shutdown(create_linode): + linode = create_linode + + wait_for_condition(10, 100, get_status, linode, "running") + + retry_sending_request(3, linode.shutdown) + + wait_for_condition(10, 100, get_status, linode, "offline") + + assert linode.status == "offline" + + +def test_linode_boot(create_linode): + linode = create_linode + + if linode.status != "offline": + retry_sending_request(3, linode.shutdown) + wait_for_condition(3, 100, get_status, linode, "offline") + retry_sending_request(3, linode.boot) + else: + retry_sending_request(3, linode.boot) + + wait_for_condition(10, 100, get_status, linode, "running") + + assert linode.status == "running" + + +def test_linode_resize(create_linode_for_long_running_tests): + linode = create_linode_for_long_running_tests + + wait_for_condition(10, 100, get_status, linode, "running") + + retry_sending_request(3, linode.resize, "g6-standard-6") + + wait_for_condition(10, 100, get_status, linode, "resizing") + + assert linode.status == "resizing" + + # Takes about 3-5 minute to resize, sometimes longer... + wait_for_condition(30, 600, get_status, linode, "running") + + assert linode.status == "running" + + +def test_linode_resize_with_class( + get_client, create_linode_for_long_running_tests +): + linode = create_linode_for_long_running_tests + ltype = Type(get_client, "g6-standard-6") + + wait_for_condition(10, 100, get_status, linode, "running") + + time.sleep(5) + res = linode.resize(new_type=ltype) + + assert res + + wait_for_condition(10, 300, get_status, linode, "resizing") + + assert linode.status == "resizing" + + # Takes about 3-5 minute to resize, sometimes longer... + wait_for_condition(30, 600, get_status, linode, "running") + + assert linode.status == "running" + + +def test_linode_boot_with_config(create_linode): + linode = create_linode + + wait_for_condition(10, 100, get_status, linode, "running") + retry_sending_request(3, linode.shutdown) + + wait_for_condition(30, 300, get_status, linode, "offline") + + config = linode.configs[0] + + retry_sending_request(3, linode.boot, config) + + wait_for_condition(10, 100, get_status, linode, "running") + + assert linode.status == "running" + + +def test_linode_firewalls(create_linode_with_volume_firewall): + linode = create_linode_with_volume_firewall + + firewalls = linode.firewalls() + + assert len(firewalls) > 0 + assert "TestSDK" in firewalls[0].label + + +def test_linode_volumes(create_linode_with_volume_firewall): + linode = create_linode_with_volume_firewall + + volumes = linode.volumes() + + assert len(volumes) > 0 + assert "TestSDK" in volumes[0].label + + +def test_linode_disk_duplicate(get_client, create_linode): + pytest.skip("Need to find out the space sizing when duplicating disks") + linode = create_linode + + disk = get_client.load(Disk, linode.disks[0].id, linode.id) + + try: + dup_disk = disk.duplicate() + assert dup_disk.linode_id == linode.id + except ApiError as e: + assert e.status == 400 + assert "Insufficient space" in str(e.json) + + +def test_linode_instance_password(create_linode_for_pass_reset): + pytest.skip("Failing due to mismatched request body") + linode = create_linode_for_pass_reset[0] + password = create_linode_for_pass_reset[1] + + wait_for_condition(10, 100, get_status, linode, "running") + + retry_sending_request(3, linode.shutdown) + + wait_for_condition(10, 200, get_status, linode, "offline") + + linode.reset_instance_root_password(root_password=password) + + linode.boot() + + wait_for_condition(10, 100, get_status, linode, "running") + + assert linode.status == "running" + + +def test_linode_ips(create_linode): + linode = create_linode + + ips = linode.ips + + assert ips.ipv4.public[0].address == linode.ipv4[0] + + +def test_linode_initate_migration(get_client): + client = get_client + available_regions = client.regions() + chosen_region = available_regions[0] + label = get_test_label() + "_migration" + + linode, password = client.linode.instance_create( + "g5-standard-4", chosen_region, image="linode/debian9", label=label + ) + + wait_for_condition(10, 100, get_status, linode, "running") + # Says it could take up to ~6 hrs for migration to fully complete + linode.initiate_migration(region="us-central") + + res = linode.delete() + + assert res + + +def test_linode_create_disk(create_linode): + pytest.skip( + "Pre-requisite for the test account need to comply with this test" + ) + linode = create_linode + disk, gen_pass = linode.disk_create() + + +def test_disk_resize(): + pytest.skip( + "Pre-requisite for the test account need to comply with this test" + ) + + +def test_config_update_interfaces(create_linode): + linode = create_linode + new_interfaces = [ + {"purpose": "public"}, + {"purpose": "vlan", "label": "cool-vlan"}, + ] + + config = linode.configs[0] + + config.interfaces = new_interfaces + + res = config.save() + + assert res + assert "cool-vlan" in str(config.interfaces) + + +def test_get_config(get_client, create_linode): + pytest.skip( + "Model get method: client.load(Config, 123, 123) does not work..." + ) + linode = create_linode + json = get_client.get( + "linode/instances/" + + str(linode.id) + + "/configs/" + + str(linode.configs[0].id) + ) + config = Config(get_client, linode.id, linode.configs[0].id, json=json) + + assert config.id == linode.configs[0].id + + +def test_get_linode_types(get_client): + types = get_client.linode.types() + + ids = [i.id for i in types] + + assert len(types) > 0 + assert "g6-nanode-1" in ids + + +def test_get_linode_type_by_id(get_client): + pytest.skip( + "Might need Type to match how other object models are behaving e.g. client.load(Type, 123)" + ) + + +def test_get_linode_type_gpu(): + pytest.skip( + "Might need Type to match how other object models are behaving e.g. client.load(Type, 123)" + ) + + +def test_save_linode_noforce(get_client, create_linode): + linode = create_linode + old_label = linode.label + linode.label = "updated_no_force_label" + linode.save(force=False) + + linode = get_client.load(Instance, linode.id) + + assert old_label != linode.label + + +def test_save_linode_force(get_client, create_linode): + linode = create_linode + old_label = linode.label + linode.label = "updated_force_label" + linode.save(force=False) + + linode = get_client.load(Instance, linode.id) + + assert old_label != linode.label diff --git a/test/integration/models/test_lke.py b/test/integration/models/test_lke.py new file mode 100644 index 000000000..094e9ae36 --- /dev/null +++ b/test/integration/models/test_lke.py @@ -0,0 +1,137 @@ +import re +from test.integration.helpers import ( + get_test_label, + send_request_when_resource_available, + wait_for_condition, +) + +import pytest + +from linode_api4.errors import ApiError +from linode_api4.objects import LKECluster, LKENodePool, LKENodePoolNode + + +@pytest.fixture(scope="session") +def create_lke_cluster(get_client): + node_type = get_client.linode.types()[1] # g6-standard-1 + version = get_client.lke.versions()[0] + region = get_client.regions().first() + node_pools = get_client.lke.node_pool(node_type, 3) + label = get_test_label() + "_cluster" + + cluster = get_client.lke.cluster_create(region, label, node_pools, version) + + yield cluster + + cluster.delete() + + +def get_cluster_status(cluster: LKECluster, status: str): + return cluster._raw_json["status"] == status + + +def get_node_status(cluster: LKECluster, status: str): + node = cluster.pools[0].nodes[0] + return node.status == status + + +def test_get_lke_clusters(get_client, create_lke_cluster): + cluster = get_client.load(LKECluster, create_lke_cluster.id) + + assert cluster._raw_json == create_lke_cluster._raw_json + + +def test_get_lke_pool(get_client, create_lke_cluster): + pytest.skip("client.load(LKENodePool, 123, 123) does not work") + + cluster = create_lke_cluster + + pool = get_client.load(LKENodePool, cluster.pools[0].id, cluster.id) + + assert cluster.pools[0]._raw_json == pool + + +def test_cluster_dashboard_url_view(create_lke_cluster): + cluster = create_lke_cluster + + url = send_request_when_resource_available( + 300, cluster.cluster_dashboard_url_view + ) + + assert re.search("https://+", url) + + +def test_kubeconfig_delete(create_lke_cluster): + cluster = create_lke_cluster + + cluster.kubeconfig_delete() + + +def test_lke_node_view(create_lke_cluster): + cluster = create_lke_cluster + node_id = cluster.pools[0].nodes[0].id + + node = cluster.node_view(node_id) + + assert node.status in ("ready", "not_ready") + assert node.id == node_id + assert node.instance_id + + +def test_lke_node_delete(create_lke_cluster): + cluster = create_lke_cluster + node_id = cluster.pools[0].nodes[0].id + + cluster.node_delete(node_id) + + with pytest.raises(ApiError) as err: + cluster.node_view(node_id) + assert "Not found" in str(err.json) + + +def test_lke_node_recycle(get_client, create_lke_cluster): + cluster = get_client.load(LKECluster, create_lke_cluster.id) + node = cluster.pools[0].nodes[0] + node_id = cluster.pools[0].nodes[0].id + + send_request_when_resource_available(300, cluster.node_recycle, node_id) + + wait_for_condition(10, 300, get_node_status, cluster, "not_ready") + + node = cluster.pools[0].nodes[0] + assert node.status == "not_ready" + + # wait for provisioning + wait_for_condition(10, 300, get_node_status, cluster, "ready") + + node = cluster.pools[0].nodes[0] + assert node.status == "ready" + + +def test_lke_cluster_nodes_recycle(get_client, create_lke_cluster): + cluster = create_lke_cluster + + send_request_when_resource_available(300, cluster.cluster_nodes_recycle) + + wait_for_condition(5, 120, get_node_status, cluster, "not_ready") + + node = cluster.pools[0].nodes[0] + assert node.status == "not_ready" + + +def test_lke_cluster_regenerate(create_lke_cluster): + pytest.skip( + "Skipping reason: '400: At least one of kubeconfig or servicetoken is required.'" + ) + cluster = create_lke_cluster + + cluster.cluster_regenerate() + + +def test_service_token_delete(create_lke_cluster): + pytest.skip( + "Skipping reason: '400: At least one of kubeconfig or servicetoken is required.'" + ) + cluster = create_lke_cluster + + cluster.service_token_delete() diff --git a/test/integration/models/test_longview.py b/test/integration/models/test_longview.py new file mode 100644 index 000000000..fcb66c609 --- /dev/null +++ b/test/integration/models/test_longview.py @@ -0,0 +1,45 @@ +import re +import time + +from linode_api4.objects import LongviewClient, LongviewSubscription + + +def test_get_longview_client(get_client, create_longview_client): + longview = get_client.load(LongviewClient, create_longview_client.id) + + assert longview.id == create_longview_client.id + + +def test_update_longview_label(get_client, create_longview_client): + longview = get_client.load(LongviewClient, create_longview_client.id) + old_label = longview.label + + label = "updated_longview_label" + + longview.label = label + + longview.save() + + assert longview.label != old_label + + +def test_delete_client(get_client, create_longview_client): + client = get_client + label = "TestSDK-longview" + longview_client = client.longview.client_create(label=label) + + time.sleep(5) + + res = longview_client.delete() + + assert res + + +def test_get_longview_subscription(get_client, create_longview_client): + subs = get_client.longview.subscriptions() + sub = get_client.load(LongviewSubscription, subs[0].id) + + assert "clients_included" in str(subs.first().__dict__) + + assert re.search("[0-9]+", str(sub.price.hourly)) + assert re.search("[0-9]+", str(sub.price.monthly)) diff --git a/test/integration/models/test_networking.py b/test/integration/models/test_networking.py new file mode 100644 index 000000000..0f1dfcd87 --- /dev/null +++ b/test/integration/models/test_networking.py @@ -0,0 +1,12 @@ +from linode_api4.objects import Firewall, IPAddress, IPv6Pool, IPv6Range + + +def test_get_networking_rules(get_client, create_firewall): + firewall = get_client.load(Firewall, create_firewall.id) + + rules = firewall.get_rules() + + assert "inbound" in str(rules) + assert "inbound_policy" in str(rules) + assert "outbound" in str(rules) + assert "outbound_policy" in str(rules) diff --git a/test/integration/models/test_nodebalancer.py b/test/integration/models/test_nodebalancer.py new file mode 100644 index 000000000..6489c0a5e --- /dev/null +++ b/test/integration/models/test_nodebalancer.py @@ -0,0 +1,119 @@ +import re + +import pytest + +from linode_api4 import ApiError +from linode_api4.objects import NodeBalancerConfig, NodeBalancerNode + + +@pytest.fixture(scope="session") +def create_linode_with_private_ip(get_client): + client = get_client + available_regions = client.regions() + chosen_region = available_regions[0] + label = "linode_with_privateip" + + linode_instance, password = client.linode.instance_create( + "g5-standard-4", + chosen_region, + image="linode/debian9", + label=label, + private_ip=True, + ) + + yield linode_instance + + linode_instance.delete() + + +@pytest.fixture(scope="session") +def create_nb_config(get_client): + client = get_client + available_regions = client.regions() + chosen_region = available_regions[0] + label = "nodebalancer_test" + + nb = client.nodebalancer_create(region=chosen_region, label=label) + + config = nb.config_create() + + yield config + + config.delete() + nb.delete() + + +def test_get_nodebalancer_config(get_client, create_nb_config): + config = get_client.load( + NodeBalancerConfig, + create_nb_config.id, + create_nb_config.nodebalancer_id, + ) + + +def test_create_nb_node( + get_client, create_nb_config, create_linode_with_private_ip +): + config = get_client.load( + NodeBalancerConfig, + create_nb_config.id, + create_nb_config.nodebalancer_id, + ) + linode = create_linode_with_private_ip + address = [a for a in linode.ipv4 if re.search("192.+", a)][0] + node = config.node_create( + "node_test", address + ":80", weight=50, mode="accept" + ) + + assert re.search("192.168.+:[0-9]+", node.address) + assert "node_test" == node.label + + +def test_get_nb_node(get_client, create_nb_config): + node = get_client.load( + NodeBalancerNode, + create_nb_config.nodes[0].id, + (create_nb_config.id, create_nb_config.nodebalancer_id), + ) + + +def test_update_nb_node(get_client, create_nb_config): + config = get_client.load( + NodeBalancerConfig, + create_nb_config.id, + create_nb_config.nodebalancer_id, + ) + node = config.nodes[0] + node.label = "ThisNewLabel" + node.weight = 50 + node.mode = "accept" + node.save() + + node_updated = get_client.load( + NodeBalancerNode, + create_nb_config.nodes[0].id, + (create_nb_config.id, create_nb_config.nodebalancer_id), + ) + + assert "ThisNewLabel" == node_updated.label + assert 50 == node_updated.weight + assert "accept" == node_updated.mode + + +def test_delete_nb_node(get_client, create_nb_config): + config = get_client.load( + NodeBalancerConfig, + create_nb_config.id, + create_nb_config.nodebalancer_id, + ) + node = config.nodes[0] + + node.delete() + + with pytest.raises(ApiError) as e: + get_client.load( + NodeBalancerNode, + create_nb_config.nodes[0].id, + (create_nb_config.id, create_nb_config.nodebalancer_id), + ) + assert "Not Found" in str(e.json) diff --git a/test/integration/models/test_tag.py b/test/integration/models/test_tag.py new file mode 100644 index 000000000..aa2596633 --- /dev/null +++ b/test/integration/models/test_tag.py @@ -0,0 +1,21 @@ +from test.integration.helpers import get_test_label + +import pytest + +from linode_api4.objects import Instance, Tag + + +@pytest.fixture +def create_tag(get_client): + unique_tag = get_test_label() + "_tag" + tag = get_client.tag_create(unique_tag) + + yield tag + + tag.delete() + + +def test_get_tag(get_client, create_tag): + tag = get_client.load(Tag, create_tag.id) + + assert tag.id == create_tag.id diff --git a/test/integration/models/test_volume.py b/test/integration/models/test_volume.py new file mode 100644 index 000000000..25114178e --- /dev/null +++ b/test/integration/models/test_volume.py @@ -0,0 +1,104 @@ +import time +from test.integration.conftest import get_token +from test.integration.helpers import ( + get_test_label, + retry_sending_request, + wait_for_condition, +) + +import pytest + +from linode_api4 import LinodeClient +from linode_api4.objects import Volume + + +@pytest.fixture(scope="session") +def create_linode_for_volume(get_client): + client = get_client + available_regions = client.regions() + chosen_region = available_regions[0] + timestamp = str(int(time.time())) + label = "TestSDK-" + timestamp + + linode_instance, password = client.linode.instance_create( + "g5-standard-4", chosen_region, image="linode/debian9", label=label + ) + + yield linode_instance + + linode_instance.delete() + + +def get_status(volume: Volume, status: str): + client = LinodeClient(token=get_token()) + volume = client.load(Volume, volume.id) + return volume.status == status + + +def test_get_volume(get_client, create_volume): + volume = get_client.load(Volume, create_volume.id) + + assert volume.id == create_volume.id + + +def test_update_volume_tag(get_client, create_volume): + volume = create_volume + tag_1 = "volume_test_tag1" + tag_2 = "volume_test_tag2" + + volume.tags = [tag_1, tag_2] + volume.save() + + volume = get_client.load(Volume, create_volume.id) + + assert [tag_1, tag_2] == volume.tags + + +def test_volume_resize(get_client, create_volume): + volume = get_client.load(Volume, create_volume.id) + + wait_for_condition(10, 100, get_status, volume, "active") + + res = retry_sending_request(5, volume.resize, 21) + + assert res + + +def test_volume_clone_and_delete(get_client, create_volume): + volume = get_client.load(Volume, create_volume.id) + label = get_test_label() + + wait_for_condition(10, 100, get_status, volume, "active") + + new_volume = retry_sending_request(5, volume.clone, label) + + assert label == new_volume.label + + res = retry_sending_request(5, new_volume.delete) + + assert res, "new volume deletion failed" + + +def test_attach_volume_to_linode( + get_client, create_volume, create_linode_for_volume +): + volume = create_volume + linode = create_linode_for_volume + + res = retry_sending_request(5, volume.attach, linode.id) + + assert res + + +def test_detach_volume_to_linode( + get_client, create_volume, create_linode_for_volume +): + volume = create_volume + linode = create_linode_for_volume + + res = retry_sending_request(5, volume.detach) + + assert res + + # time wait for volume to detach before deletion occurs + time.sleep(30) diff --git a/tox.ini b/tox.ini index 0b51a2837..adb2aab2f 100644 --- a/tox.ini +++ b/tox.ini @@ -11,6 +11,6 @@ deps = httpretty commands = python setup.py install - coverage run --source linode_api4 -m pytest + coverage run --source linode_api4 -m pytest test/objects coverage report pylint linode_api4 From b97ea632e3ad82006c7281837ee759125fff8527 Mon Sep 17 00:00:00 2001 From: Lena Garber <114949949+lgarber-akamai@users.noreply.github.com> Date: Fri, 9 Jun 2023 10:09:32 -0400 Subject: [PATCH 2/8] new: Add event polling functionality (#293) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## 📝 Description This change introduces various event-polling-related methods and classes to be reused across both official and unofficial Linode API integrations. Additionally, this change adds documentation and guide for using the new event polling system. This event polling system is derived from the event polling system implemented in the Linode Ansible Collection, but is _not_ backwards compatbile. --- docs/guides/event_polling.rst | 104 +++++++++++ docs/index.rst | 2 + docs/linode_api4/linode_client.rst | 9 + docs/linode_api4/objects/models.rst | 1 - docs/linode_api4/polling.rst | 12 ++ linode_api4/__init__.py | 1 + linode_api4/groups/__init__.py | 1 + linode_api4/groups/polling.py | 90 +++++++++ linode_api4/linode_client.py | 3 + linode_api4/polling.py | 226 +++++++++++++++++++++++ requirements.txt | 1 + setup.py | 1 + test/objects/polling_test.py | 274 ++++++++++++++++++++++++++++ 13 files changed, 724 insertions(+), 1 deletion(-) create mode 100644 docs/guides/event_polling.rst create mode 100644 docs/linode_api4/polling.rst create mode 100644 linode_api4/groups/polling.py create mode 100644 linode_api4/polling.py create mode 100644 test/objects/polling_test.py diff --git a/docs/guides/event_polling.rst b/docs/guides/event_polling.rst new file mode 100644 index 000000000..b9a782f3c --- /dev/null +++ b/docs/guides/event_polling.rst @@ -0,0 +1,104 @@ +Polling for Events +================== + +There are often situations where an API request will trigger a +long-running operation (e.g. Instance shutdown) that will run +after the request has been made. These operations are tracked +through `Linode Account Events`_ which reflect the target entity, +progress, and status of these operations. + +.. _Linode Account Events: https://www.linode.com/docs/api/account/#events-list + +There are often cases where you would like for your application to +halt until these operations have succeeded. The most reliable and +efficient way to achieve this is by using the :py:class:`EventPoller` +object. + +Polling on Basic Operations +--------------------------- + +In order to poll for an operation, we must create an :py:class:`EventPoller` +object *before* the endpoint that triggers the operation has been called. + +Assuming a :py:class:`LinodeClient` object has already been created with the name +"client" and an :py:class:`Instance` object has already been created with the name "my_instance", +an :py:class:`EventPoller` can be created using the +:meth:`LinodeClient.polling.event_poller_create(...) ` +method:: + + poller = client.polling.event_poller_create( + "linode", # The type of the target entity + "linode_shutdown", # The action to poll for + entity_id=my_instance.id, # The ID of your Linode Instance + ) + +Valid values for the `type` and `action` fields can be found in the `Events Response Documentation`_. + +.. _Events Response Documentation: https://www.linode.com/docs/api/account/#events-list__responses + +From here, we can send the request to trigger the long-running operation:: + + my_instance.shutdown() + +To wait for this operation to finish, we can call the +:meth:`poller.wait_for_next_event_finished(...) ` +method:: + + poller.wait_for_next_event_finished() + +The :py:class:`timeout` (default 240) and :py:class:`interval` (default 5) arguments can optionally be used to configure the timeout +and poll frequency for this operation. + +Bringing this together, we get the following:: + + from linode_api4 import LinodeClient, Instance + + # Construct a client + client = LinodeClient("MY_LINODE_TOKEN") + + # Fetch an existing Linode Instance + my_instance = client.load(Instance, 12345) + + # Create the event poller + poller = client.polling.event_poller_create( + "linode", # The type of the target entity + "linode_shutdown", # The action to poll for + entity_id=my_instance.id, # The ID of your Linode Instance + ) + + # Shutdown the Instance + my_instance.shutdown() + + # Wait until the event has finished + poller.wait_for_next_event_finished() + + print("Linode has been successfully shutdown!") + +Polling for an Entity to be Free +-------------------------------- + +In many cases, certain operations cannot be run until any other operations running on a resource have +been completed. To ensure these operation are run reliably and do not encounter conflicts, +you can use the +:meth:`LinodeClient.polling.wait_for_entity_free(...) ` method +to wait until a resource has no running or queued operations. + +For example:: + + # Construct a client + client = LinodeClient("MY_LINODE_TOKEN") + + # Load an existing instance + my_instance = client.load(Instance, 12345) + + # Wait until the Linode is not busy + client.polling.wait_for_entity_free( + "linode", + my_instance.id + ) + + # Boot the Instance + my_instance.boot() + +The :py:class:`timeout` (default 240) and :py:class:`interval` (default 5) arguments can optionally be used to configure the timeout +and poll frequency for this operation. diff --git a/docs/index.rst b/docs/index.rst index 6a3fc724c..5fb4ad6a3 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -32,9 +32,11 @@ Table of Contents guides/getting_started guides/core_concepts + guides/event_polling guides/oauth linode_api4/linode_client linode_api4/login_client linode_api4/objects/models + linode_api4/polling linode_api4/paginated_list linode_api4/objects/filtering diff --git a/docs/linode_api4/linode_client.rst b/docs/linode_api4/linode_client.rst index 172e00864..b87a6a18f 100644 --- a/docs/linode_api4/linode_client.rst +++ b/docs/linode_api4/linode_client.rst @@ -146,6 +146,15 @@ with buckets and objects, use the s3 API directly with a library like `boto3`_. .. _boto3: https://github.com/boto/boto3 +PollingGroup +^^^^^^^^^^^^ + +Includes methods related to account event polling. + +.. autoclass:: linode_api4.linode_client.PollingGroup + :members: + :special-members: + ProfileGroup ^^^^^^^^^^^^ diff --git a/docs/linode_api4/objects/models.rst b/docs/linode_api4/objects/models.rst index 089651613..7ea664940 100644 --- a/docs/linode_api4/objects/models.rst +++ b/docs/linode_api4/objects/models.rst @@ -139,4 +139,3 @@ Volume Models :exclude-members: api_endpoint, properties, derived_url_path, id_attribute, parent_id_name :undoc-members: :inherited-members: - diff --git a/docs/linode_api4/polling.rst b/docs/linode_api4/polling.rst new file mode 100644 index 000000000..6f5d956ea --- /dev/null +++ b/docs/linode_api4/polling.rst @@ -0,0 +1,12 @@ +Event Polling +========== + +This project exposes a framework for dynamically polling on long-running Linode Events. + +See the :doc:`Event Polling Guide<../guides/event_polling>` for more details. + +EventPoller class +------------------- + +.. autoclass:: linode_api4.EventPoller + :members: diff --git a/linode_api4/__init__.py b/linode_api4/__init__.py index bd1d6023a..b347b607d 100644 --- a/linode_api4/__init__.py +++ b/linode_api4/__init__.py @@ -4,3 +4,4 @@ from linode_api4.linode_client import LinodeClient from linode_api4.login_client import LinodeLoginClient, OAuthScopes from linode_api4.paginated_list import PaginatedList +from linode_api4.polling import EventPoller diff --git a/linode_api4/groups/__init__.py b/linode_api4/groups/__init__.py index bf815fffd..e3d7658fe 100644 --- a/linode_api4/groups/__init__.py +++ b/linode_api4/groups/__init__.py @@ -11,6 +11,7 @@ from .networking import * from .nodebalancer import * from .object_storage import * +from .polling import * from .profile import * from .region import * from .support import * diff --git a/linode_api4/groups/polling.py b/linode_api4/groups/polling.py new file mode 100644 index 000000000..3eaa0edda --- /dev/null +++ b/linode_api4/groups/polling.py @@ -0,0 +1,90 @@ +import polling + +from linode_api4.groups import Group +from linode_api4.objects.account import Event +from linode_api4.polling import EventPoller, TimeoutContext + + +class PollingGroup(Group): + """ + This group contains various helper functions for polling on Linode events. + """ + + def event_poller_create( + self, + entity_type: str, + action: str, + entity_id: int = None, + ) -> EventPoller: + """ + Creates a new instance of the EventPoller class. + + :param entity_type: The type of the entity to poll for events on. + Valid values for this field can be found here: https://www.linode.com/docs/api/account/#events-list__responses + :type entity_type: str + :param action: The action that caused the Event to poll for. + Valid values for this field can be found here: https://www.linode.com/docs/api/account/#events-list__responses + :type action: str + :param entity_id: The ID of the entity to poll for. + :type entity_id: int + :param poll_interval: The interval in seconds to wait between polls. + :type poll_interval: int + + :returns: The new EventPoller object. + :rtype: EventPoller + """ + + return EventPoller( + self.client, + entity_type, + action, + entity_id=entity_id, + ) + + def wait_for_entity_free( + self, + entity_type: str, + entity_id: int, + timeout: int = 240, + interval: int = 5, + ): + """ + Waits for all events relevant events to not be scheduled or in-progress. + + :param entity_type: The type of the entity to poll for events on. + Valid values for this field can be found here: https://www.linode.com/docs/api/account/#events-list__responses + :type entity_type: str + :param entity_id: The ID of the entity to poll for. + :type entity_id: int + :param timeout: The timeout in seconds for this polling operation. + :type timeout: int + :param interval: The interval in seconds to wait between polls. + :type interval: int + """ + + timeout_ctx = TimeoutContext(timeout_seconds=timeout) + + api_filter = { + "+order": "desc", + "+order_by": "created", + "entity.id": entity_id, + "entity.type": entity_type, + } + + def poll_func(): + events = self.client.get("/account/events", filters=api_filter)[ + "data" + ] + return all( + event["status"] not in ("scheduled", "started") + for event in events + ) + + if poll_func(): + return + + polling.poll( + poll_func, + step=interval, + timeout=timeout_ctx.seconds_remaining, + ) diff --git a/linode_api4/linode_client.py b/linode_api4/linode_client.py index a067a1755..f1c7c8e75 100644 --- a/linode_api4/linode_client.py +++ b/linode_api4/linode_client.py @@ -168,6 +168,9 @@ def __init__( #: Access methods related to Images - See :any:`ImageGroup` for more information. self.images = ImageGroup(self) + #: Access methods related to Event polling - See :any:`PollingGroup` for more information. + self.polling = PollingGroup(self) + @property def _user_agent(self): return "{}python-linode_api4/{} {}".format( diff --git a/linode_api4/polling.py b/linode_api4/polling.py new file mode 100644 index 000000000..537239635 --- /dev/null +++ b/linode_api4/polling.py @@ -0,0 +1,226 @@ +import datetime +from typing import Any, Dict, List, Optional + +import polling + +from linode_api4.objects import Event + + +class TimeoutContext: + """ + TimeoutContext should be used by polling resources to track their provisioning time. + """ + + def __init__(self, timeout_seconds=120): + self._start_time = datetime.datetime.now() + self._timeout_seconds = timeout_seconds + + def start(self, start_time=datetime.datetime.now()): + """ + Sets the timeout start time to the current time. + + :param start_time: The moment when the context started. + :type start_time: datetime + """ + self._start_time = start_time + + def extend(self, seconds: int): + """ + Extends the timeout window. + + :param seconds: The number of seconds to extend the timeout period by. + :type seconds: int + """ + self._timeout_seconds += seconds + + @property + def expired(self): + """ + Whether the current timeout period has been exceeded. + + :returns: Whether this context is expired. + :rtype: bool + """ + return self.seconds_remaining < 0 + + @property + def valid(self): + """ + Whether the current timeout period has not been exceeded. + + :returns: Whether this context is valid. + :rtype: bool + """ + return not self.expired + + @property + def seconds_remaining(self): + """ + The number of seconds until the timeout period has expired. + + :returns: The number of seconds remaining in this context. + :rtype: int + """ + return self._timeout_seconds - self.seconds_since_started + + @property + def seconds_since_started(self): + """ + The number of seconds since the timeout period started. + + :returns: The number of seconds since the context started. + :rtype: int + """ + return (datetime.datetime.now() - self._start_time).seconds + + +class EventPoller: + """ + EventPoller allows modules to dynamically poll for Linode events + """ + + def __init__( + self, + client: "LinodeClient", + entity_type: str, + action: str, + entity_id: int = None, + ): + self._client = client + self._entity_type = entity_type + self._entity_id = entity_id + self._action = action + + # Initialize with an empty cache if no entity is specified + if self._entity_id is None: + self._previous_event_cache = {} + return + + # We only want the first page of this response + result = client.get("/account/events", filters=self._build_filter()) + + self._previous_event_cache = {v["id"]: v for v in result["data"]} + + def _build_filter(self) -> Dict[str, Any]: + """Generates a filter dict to use in HTTP requests""" + return { + "+order": "asc", + "+order_by": "created", + "entity.id": self._entity_id, + "entity.type": self._entity_type, + "action": self._action, + } + + def set_entity_id(self, entity_id: int) -> None: + """ + Sets the ID of the entity to filter on. + This is useful for create operations where + the entity id might not be known in __init__. + + :param entity_id: The ID of the entity to poll for. + :type entity_id: int + """ + self._entity_id = entity_id + + def _attempt_merge_event_into_cache(self, event: Dict[str, Any]): + """ + Attempts to merge the given event into the event cache. + """ + + if event["id"] in self._previous_event_cache: + return + + self._previous_event_cache[event["id"]] = event + + def _check_has_new_event( + self, events: List[Dict[str, Any]] + ) -> Optional[Dict[str, Any]]: + """ + If a new event is found in the given list, return it. + """ + + for event in events: + # Ignore cached events + if event["id"] in self._previous_event_cache: + continue + + return event + + return None + + def wait_for_next_event( + self, timeout: int = 240, interval: int = 5 + ) -> Event: + """ + Waits for and returns the next event matching the + poller's configuration. + + :param timeout: The timeout in seconds before this polling operation will fail. + :type timeout: int + :param interval: The time in seconds to wait between polls. + :type interval: int + + :returns: The resulting event. + :rtype: Event + """ + result_event: Dict[str, Any] = {} + + def poll_func(): + new_event = self._check_has_new_event( + self._client.get( + "/account/events", filters=self._build_filter() + )["data"] + ) + + event_exists = new_event is not None + + if event_exists: + nonlocal result_event + result_event = new_event + self._attempt_merge_event_into_cache(new_event) + + return event_exists + + if poll_func(): + return Event(self._client, result_event["id"], json=result_event) + + polling.poll( + poll_func, + step=interval, + timeout=timeout, + ) + + return Event(self._client, result_event["id"], json=result_event) + + def wait_for_next_event_finished( + self, timeout: int = 240, interval: int = 5 + ) -> Event: + """ + Waits for the next event to enter status `finished` or `notification`. + + :param timeout: The timeout in seconds before this polling operation will fail. + :type timeout: int + :param interval: The time in seconds to wait between polls. + :type interval: int + + :returns: The resulting event. + :rtype: Event + """ + + timeout_ctx = TimeoutContext(timeout_seconds=timeout) + event = self.wait_for_next_event(timeout_ctx.seconds_remaining) + + def poll_func(): + event._api_get() + return event.status in ["finished", "notification"] + + if poll_func(): + return event + + polling.poll( + poll_func, + step=interval, + timeout=timeout_ctx.seconds_remaining, + ) + + return event diff --git a/requirements.txt b/requirements.txt index 3d78232b6..183da90a9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ httplib2 enum34 requests +polling>=0.3.2 \ No newline at end of file diff --git a/setup.py b/setup.py index 44a78c465..25c0700b0 100755 --- a/setup.py +++ b/setup.py @@ -121,6 +121,7 @@ def bake_version(v): install_requires=[ "requests", + "polling" ], extras_require={ diff --git a/test/objects/polling_test.py b/test/objects/polling_test.py new file mode 100644 index 000000000..b4d3a88cd --- /dev/null +++ b/test/objects/polling_test.py @@ -0,0 +1,274 @@ +import json + +import httpretty +import pytest + +from linode_api4 import LinodeClient + + +class TestPolling: + @pytest.fixture(scope="class") + def client(self): + return LinodeClient("testing", base_url="https://localhost") + + @staticmethod + def body_event_status(status: str, action: str = "linode_shutdown"): + return { + "action": action, + "entity": { + "id": 11111, + "type": "linode", + }, + "id": 123, + "status": status, + } + + @staticmethod + def body_event_list_empty(): + return {"data": [], "page": 1, "pages": 1, "results": 0} + + @staticmethod + def body_event_list_status(status: str, action="linode_shutdown"): + body = TestPolling.body_event_list_empty() + body["data"].append( + TestPolling.body_event_status(status, action=action) + ) + body["results"] = 1 + + return body + + @httpretty.activate + def test_wait_for_entity_free( + self, + client, + ): + """ + Tests that the wait_for_entity_free method works as expected. + """ + + httpretty.register_uri( + httpretty.GET, + "https://localhost/account/events", + responses=[ + httpretty.Response( + body=json.dumps(self.body_event_list_status("started")), + status=200, + ), + httpretty.Response( + body=json.dumps(self.body_event_list_status("finished")), + status=200, + ), + httpretty.Response( + body=json.dumps(self.body_event_list_status("finished")), + status=200, + ), + ], + ) + + client.polling.wait_for_entity_free( + "linode", + 11111, + 10, + 0.1, + ) + + assert len(httpretty.latest_requests()) == 2 + + @httpretty.activate + def test_wait_for_entity_free_notification( + self, + client, + ): + """ + Tests that the wait_for_entity_free method works as expected with a notification event. + """ + + httpretty.register_uri( + httpretty.GET, + "https://localhost/account/events", + responses=[ + httpretty.Response( + body=json.dumps( + self.body_event_list_status("notification") + ), + status=200, + ), + httpretty.Response( + body=json.dumps( + self.body_event_list_status("notification") + ), + status=200, + ), + ], + ) + + client.polling.wait_for_entity_free( + "linode", + 11111, + 10, + 0.1, + ) + + assert len(httpretty.latest_requests()) == 1 + + for r in httpretty.latest_requests(): + filter_header = r.headers["X-Filter"] + assert '"entity.type": "linode"' in filter_header + assert '"entity.id": 11111' in filter_header + + @httpretty.activate + def test_wait_for_event_finished( + self, + client, + ): + """ + Tests that the EventPoller.wait_for_event_finished method works as expected. + """ + + httpretty.register_uri( + httpretty.GET, + "https://localhost/account/events/123", + responses=[ + httpretty.Response( + body=json.dumps(self.body_event_status("started")), + ), + httpretty.Response( + body=json.dumps(self.body_event_status("started")), + ), + httpretty.Response( + body=json.dumps(self.body_event_status("finished")), + ), + ], + ) + + httpretty.register_uri( + httpretty.GET, + "https://localhost/account/events", + responses=[ + httpretty.Response( + body=json.dumps(self.body_event_list_empty()), + status=200, + ), + httpretty.Response( + body=json.dumps(self.body_event_list_status("started")), + status=200, + ), + httpretty.Response( + body=json.dumps(self.body_event_list_status("finished")), + status=200, + ), + ], + ) + + result = client.polling.event_poller_create( + "linode", "linode_shutdown", entity_id=11111 + ).wait_for_next_event_finished(interval=0.1) + + latest_requests = httpretty.latest_requests() + + list_requests = [ + v for v in latest_requests if v.path == "/account/events" + ] + + get_requests = [ + v for v in latest_requests if v.path == "/account/events/123" + ] + + for r in list_requests: + filter_header = r.headers["X-Filter"] + assert '"entity.type": "linode"' in filter_header + assert '"entity.id": 11111' in filter_header + + assert len(list_requests) == 2 + assert len(get_requests) == 3 + assert result.entity.id == 11111 + assert result.status == "finished" + + @httpretty.activate + def test_wait_for_event_finished_creation( + self, + client, + ): + """ + Tests that the EventPoller.wait_for_event_finished method + works as expected on newly created entities. + """ + + action = "linode_create" + + httpretty.register_uri( + httpretty.GET, + "https://localhost/account/events/123", + responses=[ + httpretty.Response( + body=json.dumps( + self.body_event_status("started", action=action) + ), + ), + httpretty.Response( + body=json.dumps( + self.body_event_status("started", action=action) + ), + ), + httpretty.Response( + body=json.dumps( + self.body_event_status("finished", action=action) + ), + ), + ], + ) + + httpretty.register_uri( + httpretty.GET, + "https://localhost/account/events", + responses=[ + httpretty.Response( + body=json.dumps(self.body_event_list_empty()), + status=200, + ), + httpretty.Response( + body=json.dumps( + self.body_event_list_status("started", action=action) + ), + status=200, + ), + httpretty.Response( + body=json.dumps( + self.body_event_list_status("finished", action=action) + ), + status=200, + ), + ], + ) + + poller = client.polling.event_poller_create( + "linode", + "linode_create", + ) + + # Pretend we created an instance here + instance_id = 11111 + + poller.set_entity_id(instance_id) + + result = poller.wait_for_next_event_finished(interval=0.1) + + latest_requests = httpretty.latest_requests() + + list_requests = [ + v for v in latest_requests if v.path == "/account/events" + ] + + get_requests = [ + v for v in latest_requests if v.path == "/account/events/123" + ] + + for r in list_requests: + filter_header = r.headers["X-Filter"] + assert '"entity.type": "linode"' in filter_header + assert '"entity.id": 11111' in filter_header + + assert len(list_requests) == 2 + assert len(get_requests) == 3 + assert result.entity.id == 11111 + assert result.status == "finished" From 88e489e95777513ad257df277b16a1a48d777bec Mon Sep 17 00:00:00 2001 From: ykim-1 <126618609+ykim-1@users.noreply.github.com> Date: Fri, 9 Jun 2023 10:36:55 -0700 Subject: [PATCH 3/8] Update README and workflow_dispatch (#295) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## 📝 Description updating README and workflow_dispatch remove /acctest command ## ✔️ How to Test Refer to Testing section in README **How do I run the relevant unit/integration tests?** ## 📷 Preview **If applicable, include a screenshot or code snippet of this change. Otherwise, please remove this section.** --- .github/workflows/e2e-test-pr-command.yml | 19 -------------- .github/workflows/e2e-test-pr.yml | 28 ++++++++++++-------- Makefile | 12 ++++++++- README.rst | 31 ++++++++++++++++++++++- 4 files changed, 58 insertions(+), 32 deletions(-) delete mode 100644 .github/workflows/e2e-test-pr-command.yml diff --git a/.github/workflows/e2e-test-pr-command.yml b/.github/workflows/e2e-test-pr-command.yml deleted file mode 100644 index 3b52a695b..000000000 --- a/.github/workflows/e2e-test-pr-command.yml +++ /dev/null @@ -1,19 +0,0 @@ -name: AccTest Command - -on: - issue_comment: - types: [created] - -jobs: - acctest-command: - runs-on: ubuntu-latest - if: ${{ github.event.issue.pull_request }} - steps: - - name: Slash Command Dispatch - uses: peter-evans/slash-command-dispatch@v1.2.0 - with: - token: ${{ secrets.GITHUB_TOKEN }} - issue-type: pull-request - commands: acctest - named-args: true - permission: write diff --git a/.github/workflows/e2e-test-pr.yml b/.github/workflows/e2e-test-pr.yml index 020874a66..9ac6f8639 100644 --- a/.github/workflows/e2e-test-pr.yml +++ b/.github/workflows/e2e-test-pr.yml @@ -1,18 +1,24 @@ on: pull_request: - repository_dispatch: - types: [acctest-command] + workflow_dispatch: + inputs: + test_path: + description: 'Enter specific test path. E.g. linode_client/test_linode_client.py, models/test_account.py' + required: false + sha: + description: 'The hash value of the commit.' + required: true + pull_request_number: + description: 'The number of the PR.' + required: false name: PR E2E Tests jobs: - # Maintainer has commented /acctest on a pull request integration-fork-ubuntu: runs-on: ubuntu-latest if: - github.event_name == 'repository_dispatch' && - github.event.client_payload.slash_command.sha != '' && - github.event.client_payload.pull_request.head.sha == github.event.client_payload.slash_command.sha + github.event_name == 'workflow_dispatch' && inputs.sha != '' steps: - uses: actions-ecosystem/action-regex-match@v2 @@ -26,7 +32,7 @@ jobs: - name: Checkout PR uses: actions/checkout@v3 with: - ref: ${{ github.event.client_payload.slash_command.sha }} + ref: ${{ inputs.sha }} - name: Update system packages run: sudo apt-get update -y @@ -47,16 +53,16 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - run: make testint + - run: make INTEGRATION_TEST_PATH="${{ inputs.test_path }}" testint if: ${{ steps.validate-tests.outputs.match == '' }} env: LINODE_CLI_TOKEN: ${{ secrets.LINODE_TOKEN }} - - uses: actions/github-script@v5 + - uses: actions/github-script@v6 id: update-check-run - if: ${{ always() }} + if: ${{ inputs.pull_request_number != '' }} env: - number: ${{ github.event.client_payload.pull_request.number }} + number: ${{ inputs.pull_request_number }} job: ${{ github.job }} conclusion: ${{ job.status }} with: diff --git a/Makefile b/Makefile index cf6c2431d..992927606 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,16 @@ PYTHON ?= python3 INTEGRATION_TEST_PATH := +TEST_CASE_COMMAND := +MODEL_COMMAND := + +ifdef TEST_CASE +TEST_CASE_COMMAND = -k $(TEST_CASE) +endif + +ifdef TEST_MODEL +MODEL_COMMAND = models/$(TEST_MODEL) +endif @PHONEY: clean clean: @@ -50,4 +60,4 @@ lint: @PHONEY: testint testint: - python3 -m pytest test/integration/ + python3 -m pytest test/integration/${INTEGRATION_TEST_PATH}${MODEL_COMMAND} ${TEST_CASE_COMMAND} diff --git a/README.rst b/README.rst index f04105bdf..d00012285 100644 --- a/README.rst +++ b/README.rst @@ -101,7 +101,7 @@ Contributing Tests ----- -Tests live in the ``tests`` directory. When invoking tests, make sure you are +Tests live in the ``test`` directory. When invoking tests, make sure you are in the root directory of this project. To run the full suite across all supported python versions, use tox_: @@ -133,6 +133,35 @@ from the api base url that should be returned, for example:: .. _tox: http://tox.readthedocs.io + +Integration Tests +----------- +Integration tests live in the ``test/integration`` directory. + +Pre-requisite +^^^^^^^^^^^^^^^^^ +Export Linode API token as `LINODE_CLI_TOKEN` before running integration tests:: + + export LINODE_TOKEN = $(your_token) + +Running the tests +^^^^^^^^^^^^^^^^^ +Run the tests locally using the make command. Run the entire test suite using command below:: + + make testint + +To run a specific package, use environment variable `INTEGRATION_TEST_PATH` with `testint` command:: + + make INTEGRATION_TEST_PATH="linode_client" testint + +To run a specific model test suite, set the environment variable `TEST_MODEL` using file name in `integration/models`:: + + make TEST_MODEL="test_account.py" testint + +Lastly to run a specific test case use environment variable `TEST_CASE` with `testint` command:: + + make TEST_CASE=test_get_domain_record testint + Documentation ------------- From f45ae85df3deaa3f50028975068a213e7d14789d Mon Sep 17 00:00:00 2001 From: ykim-1 <126618609+ykim-1@users.noreply.github.com> Date: Fri, 9 Jun 2023 10:38:12 -0700 Subject: [PATCH 4/8] Add integration tests for Linode Client retry (#294) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## 📝 Description Tests added for LinodeClient retry mechanism - Moved tests in linode_client_test.py - Added tests for other allowed methods: PUT, POST, DELETE ## ✔️ How to Test pytest test/integration/linode_client/test_retry.py **How do I run the relevant unit/integration tests?** ## 📷 Preview **If applicable, include a screenshot or code snippet of this change. Otherwise, please remove this section.** --- test/integration/linode_client/test_retry.py | 175 +++++++++++++++++++ test/linode_client_test.py | 121 +------------ 2 files changed, 176 insertions(+), 120 deletions(-) create mode 100644 test/integration/linode_client/test_retry.py diff --git a/test/integration/linode_client/test_retry.py b/test/integration/linode_client/test_retry.py new file mode 100644 index 000000000..29fd1d452 --- /dev/null +++ b/test/integration/linode_client/test_retry.py @@ -0,0 +1,175 @@ +from test.integration.conftest import get_token + +import httpretty + +from linode_api4 import ApiError, LinodeClient + +""" +Tests for retrying on intermittent errors. + +.. warning:: + This test class _does not_ follow normal testing conventions for this project, + as requests are not automatically mocked. Only add tests to this class if they + pertain to the retry logic, and make sure you mock the requests calls yourself + (or else they will make real requests and those won't work). +""" +ERROR_RESPONSES = [ + httpretty.Response( + body="{}", + status=408, + ), + httpretty.Response( + body="{}", + status=429, + ), + httpretty.Response( + body="{}", + status=200, + ), +] + + +def get_retry_client(): + client = LinodeClient(token=get_token(), base_url="https://localhost") + # sidestep the validation to do immediate retries so tests aren't slow + client.retry_rate_limit_interval = 0.1 + return client + + +@httpretty.activate +def test_get_retry_statuses(): + """ + Tests that retries work as expected on 408 and 429 responses. + """ + + httpretty.register_uri( + httpretty.GET, "https://localhost/test", responses=ERROR_RESPONSES + ) + + get_retry_client().get("/test") + + assert len(httpretty.latest_requests()) == 3 + + +@httpretty.activate +def test_put_retry_statuses(): + """ + Tests that retries work as expected on 408 and 429 responses. + """ + + httpretty.register_uri( + httpretty.PUT, "https://localhost/test", responses=ERROR_RESPONSES + ) + + get_retry_client().put("/test") + + assert len(httpretty.latest_requests()) == 3 + + +@httpretty.activate +def test_post_retry_statuses(): + httpretty.register_uri( + httpretty.POST, "https://localhost/test", responses=ERROR_RESPONSES + ) + + get_retry_client.post("/test") + + assert len(httpretty.latest_requests()) == 3 + + +@httpretty.activate +def test_delete_retry_statuses(): + httpretty.register_uri( + httpretty.DELETE, "https://localhost/test", responses=ERROR_RESPONSES + ) + + get_retry_client().delete("/test") + + assert len(httpretty.latest_requests()) == 3 + + +@httpretty.activate +def test_retry_max(): + """ + Tests that retries work as expected on 408 and 429 responses. + """ + + httpretty.register_uri( + httpretty.GET, + "https://localhost/test", + responses=[ + httpretty.Response( + body="{}", + status=408, + ), + httpretty.Response( + body="{}", + status=429, + ), + httpretty.Response( + body="{}", + status=429, + ), + ], + ) + + client = get_retry_client() + client.retry_max = 2 + + try: + client.get("/test") + except ApiError as err: + assert err.status == 429 + else: + raise RuntimeError("Expected retry error after exceeding max retries") + + assert len(httpretty.latest_requests()) == 3 + + +@httpretty.activate +def test_retry_disable(): + """ + Tests that retries can be disabled. + """ + + httpretty.register_uri( + httpretty.GET, + "https://localhost/test", + responses=[ + httpretty.Response( + body="{}", + status=408, + ), + ], + ) + + client = get_retry_client() + client.retry = False + + try: + client.get("/test") + except ApiError as e: + assert e.status == 408 + else: + raise RuntimeError("Expected 408 error to be raised") + + assert len(httpretty.latest_requests()) == 1 + + +@httpretty.activate +def test_retry_works_with_integer_interval_value(): + """ + Tests that retries work as expected on 408 and 429 responses. + """ + + httpretty.register_uri( + httpretty.GET, "https://localhost/test", responses=ERROR_RESPONSES + ) + + client = get_retry_client() + client.retry_max = 2 + client.retry_rate_limit_interval = 1 + + client.get("/test") + + assert len(httpretty.latest_requests()) == 3 diff --git a/test/linode_client_test.py b/test/linode_client_test.py index 4c0f9169c..1b4318f6f 100644 --- a/test/linode_client_test.py +++ b/test/linode_client_test.py @@ -1,11 +1,7 @@ from datetime import datetime from test.base import ClientBaseCase -from unittest import TestCase -import httpretty -import pytest - -from linode_api4 import ApiError, LinodeClient, LongviewSubscription +from linode_api4 import LongviewSubscription from linode_api4.objects.linode import Instance from linode_api4.objects.networking import IPAddress from linode_api4.objects.object_storage import ( @@ -1064,118 +1060,3 @@ def test_ipv6_ranges(self): ranges = self.client.networking.ipv6_ranges() self.assertEqual(len(ranges), 1) self.assertEqual(ranges[0].range, "2600:3c01::") - - -class LinodeClientRateLimitRetryTest(TestCase): - """ - Tests for retrying on intermittent errors. - - .. warning:: - This test class _does not_ follow normal testing conventions for this project, - as requests are not automatically mocked. Only add tests to this class if they - pertain to the retry logic, and make sure you mock the requests calls yourself - (or else they will make real requests and those won't work). - """ - - def get_retry_client(self): - client = LinodeClient("testing", base_url="https://localhost") - # sidestep the validation to do immediate retries so tests aren't slow - client.retry_rate_limit_interval = 0.1 - return client - - @httpretty.activate - def test_retry_statuses(self): - """ - Tests that retries work as expected on 408 and 429 responses. - """ - - httpretty.register_uri( - httpretty.GET, - "https://localhost/test", - responses=[ - httpretty.Response( - body="{}", - status=408, - ), - httpretty.Response( - body="{}", - status=429, - ), - httpretty.Response( - body="{}", - status=200, - ), - ], - ) - - self.get_retry_client().get("/test") - - assert len(httpretty.latest_requests()) == 3 - - @httpretty.activate - def test_retry_max(self): - """ - Tests that retries work as expected on 408 and 429 responses. - """ - - httpretty.register_uri( - httpretty.GET, - "https://localhost/test", - responses=[ - httpretty.Response( - body="{}", - status=408, - ), - httpretty.Response( - body="{}", - status=429, - ), - httpretty.Response( - body="{}", - status=429, - ), - ], - ) - - client = self.get_retry_client() - client.retry_max = 2 - - try: - client.get("/test") - except ApiError as err: - assert err.status == 429 - else: - raise RuntimeError( - "Expected retry error after exceeding max retries" - ) - - assert len(httpretty.latest_requests()) == 3 - - @httpretty.activate - def test_retry_disable(self): - """ - Tests that retries can be disabled. - """ - - httpretty.register_uri( - httpretty.GET, - "https://localhost/test", - responses=[ - httpretty.Response( - body="{}", - status=408, - ), - ], - ) - - client = self.get_retry_client() - client.retry = False - - try: - client.get("/test") - except ApiError as e: - assert e.status == 408 - else: - raise RuntimeError("Expected 408 error to be raised") - - assert len(httpretty.latest_requests()) == 1 From ccb933a87590ecf3469b1f306338e280182b75eb Mon Sep 17 00:00:00 2001 From: ykim-1 <126618609+ykim-1@users.noreply.github.com> Date: Wed, 14 Jun 2023 08:07:46 -0700 Subject: [PATCH 5/8] add pr head commit hash check (#298) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## 📝 Description Add PR head commit hash check ## ✔️ How to Test **What are the steps to reproduce the issue or verify the changes?** **How do I run the relevant unit/integration tests?** ## 📷 Preview **If applicable, include a screenshot or code snippet of this change. Otherwise, please remove this section.** --- .github/workflows/e2e-test-pr.yml | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e-test-pr.yml b/.github/workflows/e2e-test-pr.yml index 9ac6f8639..c08ec571d 100644 --- a/.github/workflows/e2e-test-pr.yml +++ b/.github/workflows/e2e-test-pr.yml @@ -24,7 +24,7 @@ jobs: - uses: actions-ecosystem/action-regex-match@v2 id: validate-tests with: - text: ${{ github.event.client_payload.slash_command.tests }} + text: ${{ inputs.test_path }} regex: '[^a-z0-9-:.\/_]' # Tests validation flags: gi @@ -34,6 +34,31 @@ jobs: with: ref: ${{ inputs.sha }} + - name: Get the hash value of the latest commit from the PR branch + uses: octokit/graphql-action@v2.x + id: commit-hash + if: ${{ inputs.pull_request_number != '' }} + with: + query: | + query PRHeadCommitHash($owner: String!, $repo: String!, $pr_num: Int!) { + repository(owner:$owner, name:$repo) { + pullRequest(number: $pr_num) { + headRef { + target { + ... on Commit { + oid + } + } + } + } + } + } + owner: ${{ github.event.repository.owner.login }} + repo: ${{ github.event.repository.name }} + pr_num: ${{ fromJSON(inputs.pull_request_number) }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Update system packages run: sudo apt-get update -y @@ -60,7 +85,7 @@ jobs: - uses: actions/github-script@v6 id: update-check-run - if: ${{ inputs.pull_request_number != '' }} + if: ${{ inputs.pull_request_number != '' && fromJson(steps.commit-hash.outputs.data).repository.pullRequest.headRef.target.oid == inputs.sha }} env: number: ${{ inputs.pull_request_number }} job: ${{ github.job }} From 121a0f4631e5dbb8daee9cec03e9345982025b53 Mon Sep 17 00:00:00 2001 From: ykim-1 <126618609+ykim-1@users.noreply.github.com> Date: Wed, 14 Jun 2023 08:14:39 -0700 Subject: [PATCH 6/8] move unit tests into separate directory and modified tox.ini (#297) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## 📝 Description Unit tests currently are in the same directory as integration tests which causes a bit of confusion. Hence moving it to a different directory ## ✔️ How to Test tox **How do I run the relevant unit/integration tests?** ## 📷 Preview **If applicable, include a screenshot or code snippet of this change. Otherwise, please remove this section.** --- test/{objects => unit}/__init__.py | 0 test/{ => unit}/base.py | 3 +-- test/{ => unit}/fixtures.py | 0 test/{ => unit}/linode_client_test.py | 2 +- test/{ => unit}/login_client_test.py | 0 test/{ => unit}/objects/account_test.py | 2 +- test/{ => unit}/objects/database_test.py | 2 +- test/{ => unit}/objects/domain_test.py | 2 +- test/{ => unit}/objects/firewall_test.py | 2 +- test/{ => unit}/objects/image_test.py | 2 +- test/{ => unit}/objects/linode_test.py | 2 +- test/{ => unit}/objects/lke_test.py | 4 ++-- test/{ => unit}/objects/longview_test.py | 2 +- test/{ => unit}/objects/mapped_object_test.py | 0 test/{ => unit}/objects/networking_test.py | 4 ++-- test/{ => unit}/objects/nodebalancers_test.py | 3 +-- test/{ => unit}/objects/object_storage_test.py | 2 +- test/{ => unit}/objects/polling_test.py | 0 test/{ => unit}/objects/profile_test.py | 2 +- test/{ => unit}/objects/region_test.py | 2 +- test/{ => unit}/objects/support_test.py | 2 +- test/{ => unit}/objects/tag_test.py | 4 ++-- test/{ => unit}/objects/volume_test.py | 2 +- test/{ => unit}/paginated_list_test.py | 0 test/{ => unit}/util_test.py | 0 tox.ini | 2 +- 26 files changed, 22 insertions(+), 24 deletions(-) rename test/{objects => unit}/__init__.py (100%) rename test/{ => unit}/base.py (99%) rename test/{ => unit}/fixtures.py (100%) rename test/{ => unit}/linode_client_test.py (99%) rename test/{ => unit}/login_client_test.py (100%) rename test/{ => unit}/objects/account_test.py (99%) rename test/{ => unit}/objects/database_test.py (99%) rename test/{ => unit}/objects/domain_test.py (98%) rename test/{ => unit}/objects/firewall_test.py (98%) rename test/{ => unit}/objects/image_test.py (98%) rename test/{ => unit}/objects/linode_test.py (99%) rename test/{ => unit}/objects/lke_test.py (97%) rename test/{ => unit}/objects/longview_test.py (98%) rename test/{ => unit}/objects/mapped_object_test.py (100%) rename test/{ => unit}/objects/networking_test.py (95%) rename test/{ => unit}/objects/nodebalancers_test.py (98%) rename test/{ => unit}/objects/object_storage_test.py (99%) rename test/{ => unit}/objects/polling_test.py (100%) rename test/{ => unit}/objects/profile_test.py (98%) rename test/{ => unit}/objects/region_test.py (93%) rename test/{ => unit}/objects/support_test.py (95%) rename test/{ => unit}/objects/tag_test.py (93%) rename test/{ => unit}/objects/volume_test.py (98%) rename test/{ => unit}/paginated_list_test.py (100%) rename test/{ => unit}/util_test.py (100%) diff --git a/test/objects/__init__.py b/test/unit/__init__.py similarity index 100% rename from test/objects/__init__.py rename to test/unit/__init__.py diff --git a/test/base.py b/test/unit/base.py similarity index 99% rename from test/base.py rename to test/unit/base.py index f1e65d8ef..95bbcd0a6 100644 --- a/test/base.py +++ b/test/unit/base.py @@ -1,12 +1,11 @@ import json +from test.unit.fixtures import TestFixtures from unittest import TestCase from mock import patch from linode_api4 import LinodeClient -from .fixtures import TestFixtures - FIXTURES = TestFixtures() diff --git a/test/fixtures.py b/test/unit/fixtures.py similarity index 100% rename from test/fixtures.py rename to test/unit/fixtures.py diff --git a/test/linode_client_test.py b/test/unit/linode_client_test.py similarity index 99% rename from test/linode_client_test.py rename to test/unit/linode_client_test.py index 1b4318f6f..b04033447 100644 --- a/test/linode_client_test.py +++ b/test/unit/linode_client_test.py @@ -1,5 +1,5 @@ from datetime import datetime -from test.base import ClientBaseCase +from test.unit.base import ClientBaseCase from linode_api4 import LongviewSubscription from linode_api4.objects.linode import Instance diff --git a/test/login_client_test.py b/test/unit/login_client_test.py similarity index 100% rename from test/login_client_test.py rename to test/unit/login_client_test.py diff --git a/test/objects/account_test.py b/test/unit/objects/account_test.py similarity index 99% rename from test/objects/account_test.py rename to test/unit/objects/account_test.py index 4648de12d..09aba9e7f 100644 --- a/test/objects/account_test.py +++ b/test/unit/objects/account_test.py @@ -1,5 +1,5 @@ from datetime import datetime -from test.base import ClientBaseCase +from test.unit.base import ClientBaseCase from linode_api4.objects import ( Account, diff --git a/test/objects/database_test.py b/test/unit/objects/database_test.py similarity index 99% rename from test/objects/database_test.py rename to test/unit/objects/database_test.py index e32368d33..d5b84cebb 100644 --- a/test/objects/database_test.py +++ b/test/unit/objects/database_test.py @@ -1,4 +1,4 @@ -from test.base import ClientBaseCase +from test.unit.base import ClientBaseCase from linode_api4 import PostgreSQLDatabase from linode_api4.objects import MySQLDatabase diff --git a/test/objects/domain_test.py b/test/unit/objects/domain_test.py similarity index 98% rename from test/objects/domain_test.py rename to test/unit/objects/domain_test.py index 805e8e7f9..64376fb37 100644 --- a/test/objects/domain_test.py +++ b/test/unit/objects/domain_test.py @@ -1,4 +1,4 @@ -from test.base import ClientBaseCase +from test.unit.base import ClientBaseCase from linode_api4.objects import Domain, DomainRecord diff --git a/test/objects/firewall_test.py b/test/unit/objects/firewall_test.py similarity index 98% rename from test/objects/firewall_test.py rename to test/unit/objects/firewall_test.py index 3b8cc280b..a46ea2750 100644 --- a/test/objects/firewall_test.py +++ b/test/unit/objects/firewall_test.py @@ -1,4 +1,4 @@ -from test.base import ClientBaseCase +from test.unit.base import ClientBaseCase from linode_api4.objects import Firewall, FirewallDevice diff --git a/test/objects/image_test.py b/test/unit/objects/image_test.py similarity index 98% rename from test/objects/image_test.py rename to test/unit/objects/image_test.py index 02f392473..2f3ac610e 100644 --- a/test/objects/image_test.py +++ b/test/unit/objects/image_test.py @@ -1,6 +1,6 @@ from datetime import datetime from io import BytesIO -from test.base import ClientBaseCase +from test.unit.base import ClientBaseCase from typing import BinaryIO from unittest.mock import patch diff --git a/test/objects/linode_test.py b/test/unit/objects/linode_test.py similarity index 99% rename from test/objects/linode_test.py rename to test/unit/objects/linode_test.py index c30e71043..d336478a7 100644 --- a/test/objects/linode_test.py +++ b/test/unit/objects/linode_test.py @@ -1,5 +1,5 @@ from datetime import datetime -from test.base import ClientBaseCase +from test.unit.base import ClientBaseCase from linode_api4.objects import Config, Disk, Image, Instance, StackScript, Type diff --git a/test/objects/lke_test.py b/test/unit/objects/lke_test.py similarity index 97% rename from test/objects/lke_test.py rename to test/unit/objects/lke_test.py index c42a3de2a..5c9902b38 100644 --- a/test/objects/lke_test.py +++ b/test/unit/objects/lke_test.py @@ -1,7 +1,7 @@ from datetime import datetime -from test.base import ClientBaseCase +from test.unit.base import ClientBaseCase -from linode_api4.objects import LKECluster, LKENodePool, LKENodePoolNode +from linode_api4.objects import LKECluster, LKENodePool class LKETest(ClientBaseCase): diff --git a/test/objects/longview_test.py b/test/unit/objects/longview_test.py similarity index 98% rename from test/objects/longview_test.py rename to test/unit/objects/longview_test.py index c5002eba8..10f3388eb 100644 --- a/test/objects/longview_test.py +++ b/test/unit/objects/longview_test.py @@ -1,5 +1,5 @@ from datetime import datetime -from test.base import ClientBaseCase +from test.unit.base import ClientBaseCase from linode_api4.objects import ( LongviewClient, diff --git a/test/objects/mapped_object_test.py b/test/unit/objects/mapped_object_test.py similarity index 100% rename from test/objects/mapped_object_test.py rename to test/unit/objects/mapped_object_test.py diff --git a/test/objects/networking_test.py b/test/unit/objects/networking_test.py similarity index 95% rename from test/objects/networking_test.py rename to test/unit/objects/networking_test.py index 51db14e48..c98beea46 100644 --- a/test/objects/networking_test.py +++ b/test/unit/objects/networking_test.py @@ -1,7 +1,7 @@ -from test.base import ClientBaseCase +from test.unit.base import ClientBaseCase from linode_api4 import ExplicitNullValue -from linode_api4.objects import Firewall, IPAddress, IPv6Pool, IPv6Range +from linode_api4.objects import Firewall, IPAddress, IPv6Range class NetworkingTest(ClientBaseCase): diff --git a/test/objects/nodebalancers_test.py b/test/unit/objects/nodebalancers_test.py similarity index 98% rename from test/objects/nodebalancers_test.py rename to test/unit/objects/nodebalancers_test.py index 822012286..a02054aa4 100644 --- a/test/objects/nodebalancers_test.py +++ b/test/unit/objects/nodebalancers_test.py @@ -1,11 +1,10 @@ -from test.base import ClientBaseCase +from test.unit.base import ClientBaseCase from linode_api4.objects import ( NodeBalancer, NodeBalancerConfig, NodeBalancerNode, ) -from linode_api4.objects.base import MappedObject class NodeBalancerConfigTest(ClientBaseCase): diff --git a/test/objects/object_storage_test.py b/test/unit/objects/object_storage_test.py similarity index 99% rename from test/objects/object_storage_test.py rename to test/unit/objects/object_storage_test.py index a2c9219e2..59317afa1 100644 --- a/test/objects/object_storage_test.py +++ b/test/unit/objects/object_storage_test.py @@ -1,5 +1,5 @@ from datetime import datetime -from test.base import ClientBaseCase +from test.unit.base import ClientBaseCase from linode_api4.objects import ( ObjectStorageACL, diff --git a/test/objects/polling_test.py b/test/unit/objects/polling_test.py similarity index 100% rename from test/objects/polling_test.py rename to test/unit/objects/polling_test.py diff --git a/test/objects/profile_test.py b/test/unit/objects/profile_test.py similarity index 98% rename from test/objects/profile_test.py rename to test/unit/objects/profile_test.py index 58fbdf125..cbe8dabd7 100644 --- a/test/objects/profile_test.py +++ b/test/unit/objects/profile_test.py @@ -1,5 +1,5 @@ from datetime import datetime -from test.base import ClientBaseCase +from test.unit.base import ClientBaseCase from linode_api4.objects import Profile, ProfileLogin, SSHKey from linode_api4.objects.profile import TrustedDevice diff --git a/test/objects/region_test.py b/test/unit/objects/region_test.py similarity index 93% rename from test/objects/region_test.py rename to test/unit/objects/region_test.py index 3a2cb62d4..5fd1ee7a3 100644 --- a/test/objects/region_test.py +++ b/test/unit/objects/region_test.py @@ -1,4 +1,4 @@ -from test.base import ClientBaseCase +from test.unit.base import ClientBaseCase from linode_api4.objects import Region diff --git a/test/objects/support_test.py b/test/unit/objects/support_test.py similarity index 95% rename from test/objects/support_test.py rename to test/unit/objects/support_test.py index 50ca0f9b9..0c1ac346a 100644 --- a/test/objects/support_test.py +++ b/test/unit/objects/support_test.py @@ -1,4 +1,4 @@ -from test.base import ClientBaseCase +from test.unit.base import ClientBaseCase from linode_api4.objects import SupportTicket diff --git a/test/objects/tag_test.py b/test/unit/objects/tag_test.py similarity index 93% rename from test/objects/tag_test.py rename to test/unit/objects/tag_test.py index a6c78efbb..137d11deb 100644 --- a/test/objects/tag_test.py +++ b/test/unit/objects/tag_test.py @@ -1,6 +1,6 @@ -from test.base import ClientBaseCase +from test.unit.base import ClientBaseCase -from linode_api4.objects import Instance, Tag +from linode_api4.objects import Tag class TagTest(ClientBaseCase): diff --git a/test/objects/volume_test.py b/test/unit/objects/volume_test.py similarity index 98% rename from test/objects/volume_test.py rename to test/unit/objects/volume_test.py index 1dd652eb4..c18ac8d89 100644 --- a/test/objects/volume_test.py +++ b/test/unit/objects/volume_test.py @@ -1,5 +1,5 @@ from datetime import datetime -from test.base import ClientBaseCase +from test.unit.base import ClientBaseCase from linode_api4.objects import Volume diff --git a/test/paginated_list_test.py b/test/unit/paginated_list_test.py similarity index 100% rename from test/paginated_list_test.py rename to test/unit/paginated_list_test.py diff --git a/test/util_test.py b/test/unit/util_test.py similarity index 100% rename from test/util_test.py rename to test/unit/util_test.py diff --git a/tox.ini b/tox.ini index adb2aab2f..e12bf1415 100644 --- a/tox.ini +++ b/tox.ini @@ -11,6 +11,6 @@ deps = httpretty commands = python setup.py install - coverage run --source linode_api4 -m pytest test/objects + coverage run --source linode_api4 -m pytest test/unit coverage report pylint linode_api4 From 7e82020341135a5a6612b23ebc9fd85074dd4e1a Mon Sep 17 00:00:00 2001 From: Zhiwei Liang <121905282+zliang-akamai@users.noreply.github.com> Date: Wed, 14 Jun 2023 11:14:53 -0400 Subject: [PATCH 7/8] Update PyPI publish action (#299) --- .github/workflows/publish-pypi.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish-pypi.yaml b/.github/workflows/publish-pypi.yaml index 0d0dff60b..594a0e617 100644 --- a/.github/workflows/publish-pypi.yaml +++ b/.github/workflows/publish-pypi.yaml @@ -33,6 +33,6 @@ jobs: LINODE_SDK_VERSION: ${{ github.event.release.tag_name }} - name: Publish the release artifacts to PyPI - uses: pypa/gh-action-pypi-publish@37f50c210e3d2f9450da2cd423303d6a14a6e29f # pin@release/v1 + uses: pypa/gh-action-pypi-publish@a56da0b891b3dc519c7ee3284aff1fad93cc8598 # pin@release/v1.8.6 with: - password: ${{ secrets.PYPI_API_TOKEN }} \ No newline at end of file + password: ${{ secrets.PYPI_API_TOKEN }} From 83e8269ddcc71754ee2708b25dabf7931be94652 Mon Sep 17 00:00:00 2001 From: Zhiwei Liang <121905282+zliang-akamai@users.noreply.github.com> Date: Thu, 22 Jun 2023 15:08:53 -0400 Subject: [PATCH 8/8] Remove directly call to setup.py (#300) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## 📝 Description Calling `setup.py` is deprecated. https://blog.ganssle.io/articles/2021/10/setup-py-deprecated.html --- Makefile | 5 ++--- README.rst | 2 +- docs/guides/getting_started.rst | 2 +- docs/index.rst | 2 +- tox.ini | 2 +- 5 files changed, 6 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 992927606..c7c46267e 100644 --- a/Makefile +++ b/Makefile @@ -20,8 +20,7 @@ clean: @PHONEY: build build: clean - $(PYTHON) setup.py sdist - $(PYTHON) setup.py bdist_wheel + $(PYTHON) -m build --wheel --sdist @PHONEY: release @@ -30,7 +29,7 @@ release: build install: clean - python3 setup.py install + python3 -m pip install . requirements: diff --git a/README.rst b/README.rst index d00012285..b7f68c537 100644 --- a/README.rst +++ b/README.rst @@ -27,7 +27,7 @@ Building from Source To build and install this package: - Clone this repository -- ``./setup.py install`` +- ``python3 -m pip install .`` Usage ===== diff --git a/docs/guides/getting_started.rst b/docs/guides/getting_started.rst index 72c671b6e..01b2a6d6c 100644 --- a/docs/guides/getting_started.rst +++ b/docs/guides/getting_started.rst @@ -18,7 +18,7 @@ If you prefer, you can clone the package from github_ and install it from source git clone git@github.com:Linode/linode_api4-python cd linode_api4 - python setup.py install + python -m pip install . Authentication -------------- diff --git a/docs/index.rst b/docs/index.rst index 5fb4ad6a3..828e7e751 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -19,7 +19,7 @@ To install from source:: git clone https://github.com/linode/linode_api4-python cd linode_api4 - python setup.py install + python -m pip install . For more information, see our :doc:`Getting Started` guide. diff --git a/tox.ini b/tox.ini index e12bf1415..707f91353 100644 --- a/tox.ini +++ b/tox.ini @@ -10,7 +10,7 @@ deps = pylint httpretty commands = - python setup.py install + python -m pip install . coverage run --source linode_api4 -m pytest test/unit coverage report pylint linode_api4