diff --git a/.github/workflows/PMM_PSMDB_PBM.yml b/.github/workflows/PMM_PSMDB_PBM.yml index 8531a75d..8c2adf71 100644 --- a/.github/workflows/PMM_PSMDB_PBM.yml +++ b/.github/workflows/PMM_PSMDB_PBM.yml @@ -5,34 +5,40 @@ on: inputs: psmdb_version: description: "psmdb version" + default: "latest" required: false pbm_version: description: "pbm version" + default: "latest" required: false pmm_version: description: "pmm2-client version" + default: "3-dev-latest" required: false pmm_image: description: "pmm-server docker image" + default: "perconalab/pmm-server:3-dev-latest" required: false push: branches: - main + - -v3 pull_request: branches: - main + - v3 jobs: test_replica_set: runs-on: ubuntu-20.04 timeout-minutes: 20 env: - PSMDB_VERSION: ${{ github.event.inputs.psmdb_version || 'latest' }} - PBM_VERSION: ${{ github.event.inputs.pbm_version || 'latest' }} - PMM_CLIENT_VERSION: ${{ github.event.inputs.pmm_version || 'latest' }} - PMM_IMAGE: ${{ github.event.inputs.pmm_image || 'perconalab/pmm-server:dev-latest' }} + PSMDB_VERSION: ${{ inputs.psmdb_version || 'latest' }} + PBM_VERSION: ${{ inputs.pbm_version || 'latest' }} + PMM_CLIENT_VERSION: ${{ inputs.pmm_version || '3-dev-latest' }} + PMM_IMAGE: ${{ inputs.pmm_image || 'perconalab/pmm-server:3-dev-latest' }} steps: - uses: actions/checkout@v2 - name: test-rs @@ -44,10 +50,10 @@ jobs: runs-on: ubuntu-20.04 timeout-minutes: 20 env: - PSMDB_VERSION: ${{ github.event.inputs.psmdb_version || 'latest' }} - PBM_VERSION: ${{ github.event.inputs.pbm_version || 'latest' }} - PMM_CLIENT_VERSION: ${{ github.event.inputs.pmm_version || 'latest' }} - PMM_IMAGE: ${{ github.event.inputs.pmm_image || 'perconalab/pmm-server:dev-latest' }} + PSMDB_VERSION: ${{ inputs.psmdb_version || 'latest' }} + PBM_VERSION: ${{ inputs.pbm_version || 'latest' }} + PMM_CLIENT_VERSION: ${{ inputs.pmm_version || '3-dev-latest' }} + PMM_IMAGE: ${{ inputs.pmm_image || 'perconalab/pmm-server:3-dev-latest' }} steps: - uses: actions/checkout@v2 - name: test-sharded @@ -59,14 +65,11 @@ jobs: runs-on: ubuntu-20.04 timeout-minutes: 20 env: - PSMDB_VERSION: ${{ github.event.inputs.psmdb_version || 'latest' }} - PBM_VERSION: ${{ github.event.inputs.pbm_version || 'latest' }} - PMM_CLIENT_VERSION: ${{ github.event.inputs.pmm_version || 'latest' }} - PMM_IMAGE: ${{ github.event.inputs.pmm_image || 'perconalab/pmm-server:dev-latest' }} + PSMDB_VERSION: ${{ inputs.psmdb_version || 'latest' }} + PBM_VERSION: ${{ inputs.pbm_version || 'latest' }} + PMM_CLIENT_VERSION: ${{ inputs.pmm_version || '3-dev-latest' }} + PMM_IMAGE: ${{ inputs.pmm_image || 'perconalab/pmm-server:3-dev-latest' }} steps: - - uses: KengoTODA/actions-setup-docker-compose@v1 - with: - version: '2.17.2' - uses: actions/checkout@v3 - name: test-auth run: | diff --git a/pmm_psmdb-pbm_setup/Dockerfile b/pmm_psmdb-pbm_setup/Dockerfile index 240a81e0..62fc3e70 100644 --- a/pmm_psmdb-pbm_setup/Dockerfile +++ b/pmm_psmdb-pbm_setup/Dockerfile @@ -52,20 +52,21 @@ RUN set -ex; \ yum -y install percona-backup-mongodb ; \ else \ yum -y install percona-backup-mongodb-${PBM_VERSION}.el9 ; \ - fi && \ - if [[ $PMM_CLIENT_VERSION == http* ]]; then \ - yum -y install pmm2-client && \ - curl -Lf -o /tmp/pmm2-client.tar.gz $PMM_CLIENT_VERSION && \ - cd /tmp && tar -xvf pmm2-client.tar.gz --transform 's,^/*[^/]*,/pmm2-client,S' && \ - cd /tmp/pmm2-client && ./install_tarball && cd ../ && rm -rf pmm2* ;\ - elif [[ "$PMM_CLIENT_VERSION" =~ latest|dev-latest ]]; then \ - yum -y install pmm2-client ; \ - elif [ $PMM_CLIENT_VERSION == "pmm2-rc" ]; then \ + fi + +RUN if [[ $PMM_CLIENT_VERSION == http* ]]; then \ + yum -y install pmm-client && \ + curl -Lf -o /tmp/pmm-client.tar.gz $PMM_CLIENT_VERSION && \ + cd /tmp && tar -xvf pmm-client.tar.gz --transform 's,^/*[^/]*,/pmm-client,S' && \ + cd /tmp/pmm-client && ./install_tarball && cd ../ && rm -rf pmm* ;\ + elif [[ "$PMM_CLIENT_VERSION" =~ 3-dev-latest|latest ]]; then \ + yum -y install pmm-client ; \ + elif [[ $PMM_CLIENT_VERSION =~ pmm3-rc ]]; then \ percona-release enable original testing \ yum update --refresh -y \ - yum -y install pmm2-client ; \ + yum -y install pmm-client ; \ else \ - yum -y install pmm2-client-${PMM_CLIENT_VERSION}-6.el9 ; \ + yum -y install pmm-client-${PMM_CLIENT_VERSION}-6.el9 ; \ fi && \ systemctl enable mongod && \ systemctl enable pbm-agent && \ diff --git a/pmm_psmdb-pbm_setup/docker-compose-pmm.yaml b/pmm_psmdb-pbm_setup/docker-compose-pmm.yaml index 6d6b23f0..0672c619 100644 --- a/pmm_psmdb-pbm_setup/docker-compose-pmm.yaml +++ b/pmm_psmdb-pbm_setup/docker-compose-pmm.yaml @@ -1,14 +1,14 @@ version: "3" services: pmm-server: - image: ${PMM_IMAGE:-perconalab/pmm-server:dev-latest} + image: ${PMM_IMAGE:-perconalab/pmm-server:3-dev-latest} container_name: pmm-server environment: - "PMM_DEBUG=1" - "ENABLE_BACKUP_MANAGEMENT=1" ports: - - "443:443" - - "8081:80" + - "443:8443" + - "8081:8080" networks: - qa-integration volumes: @@ -18,6 +18,8 @@ services: build: dockerfile: ./Dockerfile-testinfra context: . + environment: + PMM_SERVER_CONTAINER_ADDRESS: ${PMM_SERVER_CONTAINER_ADDRESS:-pmm-server:8443} volumes: - ./test:/test - /var/run/docker.sock:/var/run/docker.sock:ro diff --git a/pmm_psmdb-pbm_setup/docker-compose-rs.yaml b/pmm_psmdb-pbm_setup/docker-compose-rs.yaml index 637fb441..4495bb90 100644 --- a/pmm_psmdb-pbm_setup/docker-compose-rs.yaml +++ b/pmm_psmdb-pbm_setup/docker-compose-rs.yaml @@ -11,7 +11,7 @@ services: - PMM_REPO=${PMM_REPO:-experimental} - PBM_VERSION=${PBM_VERSION:-latest} - PSMDB_VERSION=${PSMDB_VERSION:-latest} - - PMM_CLIENT_VERSION=${PMM_CLIENT_VERSION:-latest} + - PMM_CLIENT_VERSION=${PMM_CLIENT_VERSION:-3-dev-latest} command: /bin/bash rs101: @@ -36,7 +36,7 @@ services: privileged: true environment: PBM_MONGODB_URI: mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017 - PMM_AGENT_SERVER_ADDRESS: ${PMM_SERVER_CONTAINER_ADDRESS:-pmm-server:443} + PMM_AGENT_SERVER_ADDRESS: ${PMM_SERVER_CONTAINER_ADDRESS:-pmm-server:8443} PMM_AGENT_SERVER_USERNAME: admin PMM_AGENT_SERVER_PASSWORD: ${ADMIN_PASSWORD:-password} PMM_AGENT_SERVER_INSECURE_TLS: 1 @@ -62,7 +62,7 @@ services: privileged: true environment: PBM_MONGODB_URI: mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017 - PMM_AGENT_SERVER_ADDRESS: ${PMM_SERVER_CONTAINER_ADDRESS:-pmm-server:443} + PMM_AGENT_SERVER_ADDRESS: ${PMM_SERVER_CONTAINER_ADDRESS:-pmm-server:8443} PMM_AGENT_SERVER_USERNAME: admin PMM_AGENT_SERVER_PASSWORD: ${ADMIN_PASSWORD:-password} PMM_AGENT_SERVER_INSECURE_TLS: 1 @@ -88,7 +88,7 @@ services: privileged: true environment: PBM_MONGODB_URI: mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017 - PMM_AGENT_SERVER_ADDRESS: ${PMM_SERVER_CONTAINER_ADDRESS:-pmm-server:443} + PMM_AGENT_SERVER_ADDRESS: ${PMM_SERVER_CONTAINER_ADDRESS:-pmm-server:8443} PMM_AGENT_SERVER_USERNAME: admin PMM_AGENT_SERVER_PASSWORD: ${ADMIN_PASSWORD:-password} PMM_AGENT_SERVER_INSECURE_TLS: 1 @@ -116,7 +116,7 @@ services: privileged: true environment: PBM_MONGODB_URI: mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017 - PMM_AGENT_SERVER_ADDRESS: ${PMM_SERVER_CONTAINER_ADDRESS:-pmm-server:443} + PMM_AGENT_SERVER_ADDRESS: ${PMM_SERVER_CONTAINER_ADDRESS:-pmm-server:8443} PMM_AGENT_SERVER_USERNAME: admin PMM_AGENT_SERVER_PASSWORD: ${ADMIN_PASSWORD:-password} PMM_AGENT_SERVER_INSECURE_TLS: 1 @@ -142,7 +142,7 @@ services: privileged: true environment: PBM_MONGODB_URI: mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017 - PMM_AGENT_SERVER_ADDRESS: ${PMM_SERVER_CONTAINER_ADDRESS:-pmm-server:443} + PMM_AGENT_SERVER_ADDRESS: ${PMM_SERVER_CONTAINER_ADDRESS:-pmm-server:8443} PMM_AGENT_SERVER_USERNAME: admin PMM_AGENT_SERVER_PASSWORD: ${ADMIN_PASSWORD:-password} PMM_AGENT_SERVER_INSECURE_TLS: 1 @@ -168,7 +168,7 @@ services: privileged: true environment: PBM_MONGODB_URI: mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017 - PMM_AGENT_SERVER_ADDRESS: ${PMM_SERVER_CONTAINER_ADDRESS:-pmm-server:443} + PMM_AGENT_SERVER_ADDRESS: ${PMM_SERVER_CONTAINER_ADDRESS:-pmm-server:8443} PMM_AGENT_SERVER_USERNAME: admin PMM_AGENT_SERVER_PASSWORD: ${ADMIN_PASSWORD:-password} PMM_AGENT_SERVER_INSECURE_TLS: 1 diff --git a/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml b/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml index d26b7b29..574520de 100644 --- a/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml +++ b/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml @@ -7,10 +7,10 @@ services: context: . args: - REPO=${REPO:-testing} - - PMM_REPO=${PMM_REPO:-release} + - PMM_REPO=${PMM_REPO:-experimental} - PBM_VERSION=${PBM_VERSION:-latest} - PSMDB_VERSION=${PSMDB_VERSION:-latest} - - PMM_CLIENT_VERSION=${PMM_CLIENT_VERSION:-latest} + - PMM_CLIENT_VERSION=${PMM_CLIENT_VERSION:-3-dev-latest} command: /bin/bash rs101: @@ -24,7 +24,7 @@ services: privileged: true environment: - "PBM_MONGODB_URI=mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017" - - "PMM_AGENT_SERVER_ADDRESS=pmm-server:443" + - "PMM_AGENT_SERVER_ADDRESS=pmm-server:8443" - "PMM_AGENT_SERVER_USERNAME=admin" - "PMM_AGENT_SERVER_PASSWORD=password" - "PMM_AGENT_SERVER_INSECURE_TLS=1" @@ -44,7 +44,7 @@ services: privileged: true environment: - "PBM_MONGODB_URI=mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017" - - "PMM_AGENT_SERVER_ADDRESS=pmm-server:443" + - "PMM_AGENT_SERVER_ADDRESS=pmm-server:8443" - "PMM_AGENT_SERVER_USERNAME=admin" - "PMM_AGENT_SERVER_PASSWORD=password" - "PMM_AGENT_SERVER_INSECURE_TLS=1" @@ -64,7 +64,7 @@ services: privileged: true environment: - "PBM_MONGODB_URI=mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017" - - "PMM_AGENT_SERVER_ADDRESS=pmm-server:443" + - "PMM_AGENT_SERVER_ADDRESS=pmm-server:8443" - "PMM_AGENT_SERVER_USERNAME=admin" - "PMM_AGENT_SERVER_PASSWORD=password" - "PMM_AGENT_SERVER_INSECURE_TLS=1" @@ -84,7 +84,7 @@ services: privileged: true environment: - "PBM_MONGODB_URI=mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017" - - "PMM_AGENT_SERVER_ADDRESS=pmm-server:443" + - "PMM_AGENT_SERVER_ADDRESS=pmm-server:8443" - "PMM_AGENT_SERVER_USERNAME=admin" - "PMM_AGENT_SERVER_PASSWORD=password" - "PMM_AGENT_SERVER_INSECURE_TLS=1" @@ -104,7 +104,7 @@ services: privileged: true environment: - "PBM_MONGODB_URI=mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017" - - "PMM_AGENT_SERVER_ADDRESS=pmm-server:443" + - "PMM_AGENT_SERVER_ADDRESS=pmm-server:8443" - "PMM_AGENT_SERVER_USERNAME=admin" - "PMM_AGENT_SERVER_PASSWORD=password" - "PMM_AGENT_SERVER_INSECURE_TLS=1" @@ -124,7 +124,7 @@ services: privileged: true environment: - "PBM_MONGODB_URI=mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017" - - "PMM_AGENT_SERVER_ADDRESS=pmm-server:443" + - "PMM_AGENT_SERVER_ADDRESS=pmm-server:8443" - "PMM_AGENT_SERVER_USERNAME=admin" - "PMM_AGENT_SERVER_PASSWORD=password" - "PMM_AGENT_SERVER_INSECURE_TLS=1" @@ -144,7 +144,7 @@ services: privileged: true environment: - "PBM_MONGODB_URI=mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017" - - "PMM_AGENT_SERVER_ADDRESS=pmm-server:443" + - "PMM_AGENT_SERVER_ADDRESS=pmm-server:8443" - "PMM_AGENT_SERVER_USERNAME=admin" - "PMM_AGENT_SERVER_PASSWORD=password" - "PMM_AGENT_SERVER_INSECURE_TLS=1" @@ -164,7 +164,7 @@ services: privileged: true environment: - "PBM_MONGODB_URI=mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017" - - "PMM_AGENT_SERVER_ADDRESS=pmm-server:443" + - "PMM_AGENT_SERVER_ADDRESS=pmm-server:8443" - "PMM_AGENT_SERVER_USERNAME=admin" - "PMM_AGENT_SERVER_PASSWORD=password" - "PMM_AGENT_SERVER_INSECURE_TLS=1" @@ -184,7 +184,7 @@ services: privileged: true environment: - "PBM_MONGODB_URI=mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017" - - "PMM_AGENT_SERVER_ADDRESS=pmm-server:443" + - "PMM_AGENT_SERVER_ADDRESS=pmm-server:8443" - "PMM_AGENT_SERVER_USERNAME=admin" - "PMM_AGENT_SERVER_PASSWORD=password" - "PMM_AGENT_SERVER_INSECURE_TLS=1" @@ -237,14 +237,14 @@ services: /bin/sh -c " sleep 5; /usr/bin/mc config host add myminio http://minio:9000 minio1234 minio1234; /usr/bin/mc mb myminio/bcp; exit 0; " pmm-server: - image: ${PMM_IMAGE:-perconalab/pmm-server:dev-latest} + image: ${PMM_IMAGE:-perconalab/pmm-server:3-dev-latest} container_name: pmm-server environment: - "PMM_DEBUG=1" - "ENABLE_BACKUP_MANAGEMENT=1" ports: - - "443:443" - - "8081:80" + - "443:8443" + - "8081:8080" networks: - test-network volumes: @@ -254,6 +254,8 @@ services: build: dockerfile: ./Dockerfile-testinfra context: . + environment: + PMM_SERVER_CONTAINER_ADDRESS: ${PMM_SERVER_CONTAINER_ADDRESS:-pmm-server:8443} volumes: - ./test:/test - /var/run/docker.sock:/var/run/docker.sock:ro diff --git a/pmm_psmdb-pbm_setup/test/test.py b/pmm_psmdb-pbm_setup/test/test.py index 77be1be6..a748fa84 100755 --- a/pmm_psmdb-pbm_setup/test/test.py +++ b/pmm_psmdb-pbm_setup/test/test.py @@ -1,5 +1,6 @@ +import os + import requests -import docker import pytest import testinfra import time @@ -8,7 +9,9 @@ docker_rs101 = testinfra.get_host('docker://rs101') docker_rs102 = testinfra.get_host('docker://rs102') docker_rs103 = testinfra.get_host('docker://rs103') -testinfra_hosts = ['docker://rs101','docker://rs102','docker://rs103'] +testinfra_hosts = ['docker://rs101', 'docker://rs102', 'docker://rs103'] + +pmm_server_url = os.getenv('PMM_SERVER_CONTAINER_ADDRESS') pytest.location_id = '' pytest.service_id = '' @@ -18,8 +21,10 @@ pytest.pbm_backup_name = '' pytest.restore_id = '' + def test_pmm_services(): - req = requests.post('https://pmm-server/v1/inventory/Services/List',json={},headers = {"authorization": "Basic YWRtaW46cGFzc3dvcmQ="},verify=False) + req = requests.post(f"https://{pmm_server_url}/v1/inventory/Services/List", json={}, + headers={"authorization": "Basic YWRtaW46cGFzc3dvcmQ="}, verify=False) print('\nGetting all mongodb services:') mongodb = req.json()['mongodb'] print(mongodb) @@ -28,27 +33,30 @@ def test_pmm_services(): for service in mongodb: assert "rs" or "mongos" in service['service_name'] if not "mongos" in service['service_name']: - pytest.service_id = service['service_id'] + pytest.service_id = service['service_id'] print('This service_id will be used in the next steps') print(pytest.service_id) + def test_pmm_add_location(): data = { 'name': 'test', 'description': 'test', 's3_config': { - 'endpoint': 'http://minio:9000', - 'access_key': 'minio1234', - 'secret_key': 'minio1234', - 'bucket_name': 'bcp' - } + 'endpoint': 'http://minio:9000', + 'access_key': 'minio1234', + 'secret_key': 'minio1234', + 'bucket_name': 'bcp' } - req = requests.post('https://pmm-server/v1/management/backup/Locations/Add',json=data,headers = {"authorization": "Basic YWRtaW46cGFzc3dvcmQ="},verify=False) + } + req = requests.post(f"https://{pmm_server_url}/v1/management/backup/Locations/Add", json=data, + headers={"authorization": "Basic YWRtaW46cGFzc3dvcmQ="}, verify=False) print('\nAdding new location:') print(req.json()) assert "location_id" in req.json()['location_id'] pytest.location_id = req.json()['location_id'] + def test_pmm_logical_backup(): data = { 'service_id': pytest.service_id, @@ -57,18 +65,21 @@ def test_pmm_logical_backup(): 'description': 'test', 'retries': 0, 'data_model': 'LOGICAL' - } - req = requests.post('https://pmm-server/v1/management/backup/Backups/Start',json=data,headers = {"authorization": "Basic YWRtaW46cGFzc3dvcmQ="},verify=False) + } + req = requests.post(f"https://{pmm_server_url}/v1/management/backup/Backups/Start", json=data, + headers={"authorization": "Basic YWRtaW46cGFzc3dvcmQ="}, verify=False) print('\nCreating logical backup:') print(req.json()) assert "artifact_id" in req.json()['artifact_id'] pytest.artifact_id = req.json()['artifact_id'] + def test_pmm_artifact(): backup_complete = False for i in range(600): done = False - req = requests.post('https://pmm-server/v1/management/backup/Artifacts/List',json={},headers = {"authorization": "Basic YWRtaW46cGFzc3dvcmQ="},verify=False) + req = requests.post(f"https://{pmm_server_url}/v1/management/backup/Artifacts/List", json={}, + headers={"authorization": "Basic YWRtaW46cGFzc3dvcmQ="}, verify=False) assert req.json()['artifacts'] for artifact in req.json()['artifacts']: if artifact['artifact_id'] == pytest.artifact_id: @@ -80,7 +91,7 @@ def test_pmm_artifact(): print(artifact) pytest.artifact_pbm_meta = artifact['metadata_list'][0]['pbm_metadata']['name'] if "is_sharded_cluster" in artifact: - pytest.artifact_is_sharded = artifact['is_sharded_cluster'] + pytest.artifact_is_sharded = artifact['is_sharded_cluster'] break if done: backup_complete = True @@ -89,6 +100,7 @@ def test_pmm_artifact(): time.sleep(1) assert backup_complete + def test_pbm_artifact(): status = docker_rs101.check_output('pbm status --out json') parsed_status = json.loads(status) @@ -98,26 +110,30 @@ def test_pbm_artifact(): assert parsed_status['backups']['snapshot'][0]['status'] == "done" pytest.pbm_backup_name = parsed_status['backups']['snapshot'][0]['name'] + def test_pmm_start_restore(): if pytest.artifact_is_sharded == True: pytest.skip("Unsupported setup for restore from UI") data = { 'service_id': pytest.service_id, 'artifact_id': pytest.artifact_id - } - req = requests.post('https://pmm-server/v1/management/backup/Backups/Restore',json=data,headers = {"authorization": "Basic YWRtaW46cGFzc3dvcmQ="},verify=False) + } + req = requests.post(f"https://{pmm_server_url}/v1/management/backup/Backups/Restore", json=data, + headers={"authorization": "Basic YWRtaW46cGFzc3dvcmQ="}, verify=False) print('\nRestoring logical backup:') print(req.json()) assert "restore_id" in req.json()['restore_id'] pytest.restore_id = req.json()['restore_id'] + def test_pmm_restore(): if pytest.artifact_is_sharded == True: pytest.skip("Unsupported setup for restore from UI") restore_complete = False for i in range(600): done = False - req = requests.post('https://pmm-server/v1/management/backup/RestoreHistory/List',json={},headers = {"authorization": "Basic YWRtaW46cGFzc3dvcmQ="},verify=False) + req = requests.post(f"https://{pmm_server_url}/v1/management/backup/RestoreHistory/List", json={}, + headers={"authorization": "Basic YWRtaW46cGFzc3dvcmQ="}, verify=False) assert req.json()['items'] for item in req.json()['items']: if item['restore_id'] == pytest.restore_id: @@ -135,6 +151,7 @@ def test_pmm_restore(): time.sleep(1) assert restore_complete + def test_pbm_restore(): if pytest.artifact_is_sharded == True: pytest.skip("Unsupported setup for restore from UI") diff --git a/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml b/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml index 0fc4565d..b02000da 100644 --- a/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml +++ b/pmm_psmdb_diffauth_setup/docker-compose-pmm-psmdb.yml @@ -10,7 +10,7 @@ services: - PMM_REPO=${PMM_REPO:-experimental} - PBM_VERSION=${PBM_VERSION:-latest} - PSMDB_VERSION=${PSMDB_VERSION:-latest} - - PMM_CLIENT_VERSION=${PMM_CLIENT_VERSION:-latest} + - PMM_CLIENT_VERSION=${PMM_CLIENT_VERSION:-3-dev-latest} command: /bin/bash test: @@ -34,7 +34,7 @@ services: - PBM_MONGODB_URI=mongodb://${PBM_USER:-pbm}:${PBM_PASS:-pbmpass}@127.0.0.1:27017 - PMM_AGENT_SERVER_USERNAME=admin - PMM_AGENT_SERVER_PASSWORD=admin - - PMM_AGENT_SERVER_ADDRESS=pmm-server:443 + - PMM_AGENT_SERVER_ADDRESS=pmm-server:8443 entrypoint: - bash - -c @@ -62,12 +62,17 @@ services: pmm-server: container_name: pmm-server hostname: pmm-server - image: ${PMM_IMAGE:-perconalab/pmm-server:dev-latest} + image: ${PMM_IMAGE:-perconalab/pmm-server:3-dev-latest} + healthcheck: + test: [ "CMD", "curl", "-f", "http://127.0.0.1:8080/ping" ] + interval: 3s + timeout: 2s + retries: 20 environment: - "PMM_DEBUG=1" ports: - - "443:443" - - "8081:80" + - "443:8443" + - "8081:8080" volumes: - ./certs:/srv/nginx - 'pmm-server-data:/srv' diff --git a/pmm_psmdb_diffauth_setup/test/test.py b/pmm_psmdb_diffauth_setup/test/test.py index 0598adde..8ac14baa 100755 --- a/pmm_psmdb_diffauth_setup/test/test.py +++ b/pmm_psmdb_diffauth_setup/test/test.py @@ -2,17 +2,16 @@ import pytest import testinfra import re -import datetime import time import os import json -import requests -env_vars = ['AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY','AWS_USERNAME'] +env_vars = ['AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY', 'AWS_USERNAME'] client = docker.from_env() docker_pmm_client = testinfra.get_host('docker://psmdb-server') + def run_test(add_db_command): try: docker_pmm_client.check_output('pmm-admin remove mongodb psmdb-server', timeout=30) @@ -26,67 +25,74 @@ def run_test(add_db_command): pmm_admin_list = json.loads(docker_pmm_client.check_output('pmm-admin list --json', timeout=30)) for agent in pmm_admin_list['agent']: - if agent['agent_type'] == 'MONGODB_EXPORTER': - agent_id = agent['agent_id'] - agent_port = agent['port'] - break + if agent['agent_type'] == 'MONGODB_EXPORTER': + agent_id = agent['agent_id'] + agent_port = agent['port'] + break - url = f'http://psmdb-server:{agent_port}/metrics' + url = f'http://localhost:{agent_port}/metrics' try: - response = requests.get(url, auth=('pmm', agent_id), timeout=5) - assert response.status_code == 200, f"Request for metrics failed with status code {response.status_code}" + response = docker_pmm_client.check_output(f"curl --request GET --url {url} --header 'Content-Type: " + f"application/json' --user 'pmm:{agent_id}'") pattern = r'mongodb_up (\d+)' - result = re.search(pattern, response.text) + result = re.search(pattern, response) assert result is not None, "MongoDB related data isn't exported" - except requests.exceptions.ConnectionError: + except AssertionError: pytest.fail(f"Connection to {url} failed") + def test_simple_auth_wo_tls(): - run_test('pmm-admin add mongodb psmdb-server --username=pmm_mongodb --password="5M](Q%q/U+YQ<^m" '\ - '--host psmdb-server --port 27017') + run_test('pmm-admin add mongodb psmdb-server --username=pmm_mongodb --password="5M](Q%q/U+YQ<^m" ''--host ' + 'psmdb-server --port 27017') + def test_simple_auth_tls(): - run_test('pmm-admin add mongodb psmdb-server --username=pmm_mongodb --password="5M](Q%q/U+YQ<^m" '\ - '--host psmdb-server --port 27017 '\ - '--tls --tls-certificate-key-file=/mongodb_certs/client.pem --tls-ca-file=/mongodb_certs/ca-certs.pem '\ - '--cluster=mycluster') + run_test('pmm-admin add mongodb psmdb-server --username=pmm_mongodb --password="5M](Q%q/U+YQ<^m" ' + '--host psmdb-server --port 27017 ' + '--tls --tls-certificate-key-file=/mongodb_certs/client.pem --tls-ca-file=/mongodb_certs/ca-certs.pem ' + '--cluster=mycluster') + def test_x509_auth(): - run_test('pmm-admin add mongodb psmdb-server --host=psmdb-server --port 27017 '\ - '--tls --tls-certificate-key-file=/mongodb_certs/client.pem --tls-ca-file=/mongodb_certs/ca-certs.pem '\ - '--authentication-mechanism=MONGODB-X509 --authentication-database=\'$external\' '\ + run_test('pmm-admin add mongodb psmdb-server --host=psmdb-server --port 27017 ' + '--tls --tls-certificate-key-file=/mongodb_certs/client.pem --tls-ca-file=/mongodb_certs/ca-certs.pem ' + '--authentication-mechanism=MONGODB-X509 --authentication-database=\'$external\' ' '--cluster=mycluster') + def test_ldap_auth_wo_tls(): - run_test('pmm-admin add mongodb psmdb-server --username="CN=pmm-test" --password=password1 '\ - '--host=psmdb-server --port 27017 '\ - '--authentication-mechanism=PLAIN --authentication-database=\'$external\' '\ + run_test('pmm-admin add mongodb psmdb-server --username="CN=pmm-test" --password=password1 ' + '--host=psmdb-server --port 27017 ' + '--authentication-mechanism=PLAIN --authentication-database=\'$external\' ' '--cluster=mycluster') + def test_ldap_auth_tls(): - run_test('pmm-admin add mongodb psmdb-server --username="CN=pmm-test" --password=password1 '\ - '--host=psmdb-server --port 27017 '\ - '--authentication-mechanism=PLAIN --authentication-database=\'$external\' '\ - '--tls --tls-certificate-key-file=/mongodb_certs/client.pem --tls-ca-file=/mongodb_certs/ca-certs.pem '\ + run_test('pmm-admin add mongodb psmdb-server --username="CN=pmm-test" --password=password1 ' + '--host=psmdb-server --port 27017 ' + '--authentication-mechanism=PLAIN --authentication-database=\'$external\' ' + '--tls --tls-certificate-key-file=/mongodb_certs/client.pem --tls-ca-file=/mongodb_certs/ca-certs.pem ' '--cluster=mycluster') + @pytest.mark.skipif( any(not os.environ.get(var) for var in env_vars) or os.environ.get('SKIP_AWS_TESTS') == 'true', reason=f"One or more of AWS env var isn't defined or SKIP_AWS_TESTS is set to true") def test_aws_auth_wo_tls(): - run_test('pmm-admin add mongodb psmdb-server --username='+ os.environ.get('AWS_ACCESS_KEY_ID') +' '\ - '--password='+ os.environ.get('AWS_SECRET_ACCESS_KEY') +' '\ - '--host=psmdb-server --port 27017 '\ - '--authentication-mechanism=MONGODB-AWS --authentication-database=\'$external\' '\ + run_test(f'pmm-admin add mongodb psmdb-server --username={os.environ.get("AWS_ACCESS_KEY_ID")}' + f'--password={os.environ.get("AWS_SECRET_ACCESS_KEY")}' + '--host=psmdb-server --port 27017 ' + '--authentication-mechanism=MONGODB-AWS --authentication-database=\'$external\' ' '--cluster=mycluster') + @pytest.mark.skipif( any(not os.environ.get(var) for var in env_vars) or os.environ.get('SKIP_AWS_TESTS') == 'true', reason=f"One or more of AWS env var isn't defined or SKIP_AWS_TESTS is set to true") def test_aws_auth_tls(): - run_test('pmm-admin add mongodb psmdb-server --username='+ os.environ.get('AWS_ACCESS_KEY_ID') +' '\ - '--password='+ os.environ.get('AWS_SECRET_ACCESS_KEY') +' '\ - '--host=psmdb-server --port 27017 '\ - '--authentication-mechanism=MONGODB-AWS --authentication-database=\'$external\' '\ - '--tls --tls-certificate-key-file=/mongodb_certs/client.pem --tls-ca-file=/mongodb_certs/ca-certs.pem '\ + run_test(f'pmm-admin add mongodb psmdb-server --username={os.environ.get("AWS_ACCESS_KEY_ID")}' + f'--password={os.environ.get("AWS_SECRET_ACCESS_KEY")}' + '--host=psmdb-server --port 27017 ' + '--authentication-mechanism=MONGODB-AWS --authentication-database=\'$external\' ' + '--tls --tls-certificate-key-file=/mongodb_certs/client.pem --tls-ca-file=/mongodb_certs/ca-certs.pem ' '--cluster=mycluster')