Enable azure tests

closes #501

CHANGES/501.feature

@@ -0,0 +1 @@
+Enable azure tests

README.md

@@ -53,7 +53,7 @@ The following settings are stored in `template_config.yml`.
                         bindings (optional): boolean whether or not generate bindings
                         org (optional): github organization
                         revision (optional): the git commit hash to install.
-                                             "Required PR" in a commit message will override this. 
+                                             "Required PR" in a commit message will override this.
                         branch: the git branch of the plugin. applies to non-tagged GHA jobs, such as PRs & cron jobs.
                         pip_version_specifier: A pip version specifier for when the gets installed from PyPI.
                                                Applies to TAGGED (release) jobs.
@@ -83,7 +83,7 @@ The following settings are stored in `template_config.yml`.
 
   check_stray_pulpcore_imports
                         Check that plugins are importing from pulpcore.plugin and not pulpcore directly.
-  
+
   core_import_allowed   A list of string patterns to be allowed to import from pulpcore explicitly.
 
   coverage              Include collection of coverage and reporting to coveralls.io
@@ -231,6 +231,9 @@ The following settings are stored in `template_config.yml`.
                         see if it is compatible with pulpcore's master branch. This helps ensure
                         that pulpcore is following the deprecation policy for the plugin API.
 
+  test_azure            Include azure job for running tests using [azurite](https://github.com/Azure/Azurite)
+                        to emulate Azure.
+
   test_s3               Include s3 job for running tests using [minio](https://github.com/minio/minio)
                         to emulate S3.
 

plugin-template

@@ -66,6 +66,7 @@ DEFAULT_SETTINGS = {
     "test_cli": False,
     "test_performance": False,
     "test_released_plugin_with_next_pulpcore_release": False,
+    "test_azure": False,
     "test_s3": False,
     "update_redmine": False,
     "ci_trigger": "{pull_request: {branches: ['*']}}",

templates/github/.ci/ansible/Containerfile.j2.copy

@@ -14,6 +14,9 @@ RUN pip3 install \
 {%- if s3_test | default(false) -%}
 {{ " " }}django-storages[boto3] git+https://github.com/fabricio-aguiar/botocore.git@fix-100-continue
 {%- endif -%}
+{%- if azure_test | default(false) -%}
+{{ " " }}django-storages[azure]
+{%- endif -%}
 {%- for item in plugins -%}
 {%- if item.name == "pulp-certguard" -%}
 {{ " " }}python-dateutil rhsm

templates/github/.ci/ansible/settings.py.j2.copy

@@ -23,4 +23,17 @@ AWS_STORAGE_BUCKET_NAME = "pulp3"
 AWS_S3_ENDPOINT_URL = "http://minio:9000"
 DEFAULT_FILE_STORAGE = "storages.backends.s3boto3.S3Boto3Storage"
 AWS_DEFAULT_ACL = "@none None"
+MEDIA_ROOT = ""
+{% endif %}
+
+{% if azure_test | default(false) %}
+AZURE_ACCOUNT_KEY = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
+AZURE_ACCOUNT_NAME = "devstoreaccount1"
+AZURE_CONTAINER =  "pulp-test"
+AZURE_LOCATION = "pulp3"
+AZURE_OVERWRITE_FILES = True
+AZURE_URL_EXPIRATION_SECS = 120
+AZURE_CONNECTION_STRING = 'DefaultEndpointsProtocol={{ pulp_scheme }};AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint={{ pulp_scheme }}://pulp-azurite:10000/devstoreaccount1;'
+DEFAULT_FILE_STORAGE = "storages.backends.azure_storage.AzureStorage"
+MEDIA_ROOT = ""
 {% endif %}

templates/github/.github/workflows/ci.yml.j2

@@ -98,6 +98,9 @@ jobs:
           {%- if docs_test %}
           - TEST: docs
           {%- endif %}
+          {%- if test_azure %}
+          - TEST: azure
+          {%- endif %}
           {%- if test_s3 %}
           - TEST: s3
           {%- endif %}

templates/github/.github/workflows/nightly.yml.j2

@@ -30,6 +30,9 @@ jobs:
           {%- if docs_test %}
           - TEST: docs
           {%- endif %}
+          {%- if test_azure %}
+          - TEST: azure
+          {%- endif %}
           {%- if test_s3 %}
           - TEST: s3
           {%- endif %}

templates/github/.github/workflows/release.yml.j2

@@ -1,14 +1,14 @@
 {% include 'header.j2' %}
-{% from 'macros.j2' import 
-  set_env_vars, 
-  checkout, 
-  setup_python, 
+{% from 'macros.j2' import
+  set_env_vars,
+  checkout,
+  setup_python,
   setup_ruby,
-  install_httpie, 
-  after_failure, 
-  run_script, 
+  install_httpie,
+  after_failure,
+  run_script,
   set_secrets,
-  install_python_deps, 
+  install_python_deps,
   configure_git,
 with context %}
 ---
@@ -70,6 +70,9 @@ jobs:
           {%- if docs_test %}
           - TEST: docs
           {%- endif %}
+          {%- if test_azure %}
+          - TEST: azure
+          {%- endif %}
           {%- if test_s3 %}
           - TEST: s3
           {%- endif %}

templates/github/.github/workflows/scripts/before_script.sh.j2

@@ -24,7 +24,7 @@ tail -v -n +1 .ci/ansible/vars/main.yaml
 echo "PULP CONFIG:"
 tail -v -n +1 .ci/ansible/settings/settings.* ~/.config/pulp_smash/settings.json
 
-if [[ "$TEST" == 'pulp' || "$TEST" == 'performance' || "$TEST" == 'upgrade' || "$TEST" == 's3' || "$TEST" == "plugin-from-pypi" ]]; then
+if [[ "$TEST" == 'pulp' || "$TEST" == 'performance' || "$TEST" == 'upgrade' || "$TEST" == 's3' || "$TEST" == 'azure' || "$TEST" == "plugin-from-pypi" ]]; then
   # Many functional tests require these
   cmd_prefix dnf install -yq lsof which dnf-plugins-core
 fi

templates/github/.github/workflows/scripts/install.sh.j2

@@ -123,7 +123,7 @@ VARSYAML
 
 {%- if docker_fixtures %}
 
-if [[ "$TEST" == "pulp" || "$TEST" == "performance" || "$TEST" == "upgrade" || "$TEST" == "s3" || "$TEST" == "plugin-from-pypi" ]]; then
+if [[ "$TEST" == "pulp" || "$TEST" == "performance" || "$TEST" == "upgrade" || "$TEST" == "azure" || "$TEST" == "s3" || "$TEST" == "plugin-from-pypi" ]]; then
   sed -i -e '/^services:/a \
   - name: pulp-fixtures\
     image: docker.io/pulp/pulp-fixtures:latest\
@@ -149,6 +149,25 @@ minio_secret_key: "'$MINIO_SECRET_KEY'"' vars/main.yaml
 fi
 {%- endif %}
 
+{%- if test_azure %}
+
+if [ "$TEST" = "azure" ]; then
+  mkdir -p azurite
+  cd azurite
+  openssl req -newkey rsa:2048 -x509 -nodes -keyout azkey.pem -new -out azcert.pem -sha256 -days 365 -addext "subjectAltName=DNS:pulp-azurite" -subj "/C=CO/ST=ST/L=LO/O=OR/OU=OU/CN=CN"
+  sudo cp azcert.pem /usr/local/share/ca-certificates/azcert.crt
+  sudo dpkg-reconfigure ca-certificates
+  cd ..
+  sed -i -e '/^services:/a \
+  - name: pulp-azurite\
+    image: mcr.microsoft.com/azure-storage/azurite\
+    volumes:\
+      - ./azurite:/etc/pulp\
+    command: "azurite-blob --blobHost 0.0.0.0{% if pulp_scheme == "https" %} --cert /etc/pulp/azcert.pem --key /etc/pulp/azkey.pem{% endif %}"' vars/main.yaml
+  sed -i -e '$a azure_test: true' vars/main.yaml
+fi
+{%- endif %}
+
 ansible-playbook build_container.yaml
 ansible-playbook start_container.yaml
 
@@ -160,6 +179,9 @@ sudo docker cp pulp:/etc/pulp/certs/pulp_webserver.crt /usr/local/share/ca-certi
 # Hack: adding pulp CA to certifi.where()
 CERTIFI=$(python -c 'import certifi; print(certifi.where())')
 cat /usr/local/share/ca-certificates/pulp_webserver.crt | sudo tee -a $CERTIFI
+if [ "$TEST" = "azure" ]; then
+  cat /usr/local/share/ca-certificates/azcert.crt | sudo tee -a $CERTIFI
+fi
 
 # Hack: adding pulp CA to default CA file
 CERT=$(python -c 'import ssl; print(ssl.get_default_verify_paths().openssl_cafile)')
@@ -170,6 +192,16 @@ sudo update-ca-certificates
 echo ::endgroup::
 {%- endif %}
 
+if [ "$TEST" = "azure" ]; then
+  {%- if pulp_scheme == "https" %}
+  cat /usr/local/share/ca-certificates/azcert.crt >> /opt/az/lib/python3.6/site-packages/certifi/cacert.pem
+  cat /usr/local/share/ca-certificates/azcert.crt | docker exec -i pulp bash -c "cat >> /usr/local/lib/python3.8/site-packages/certifi/cacert.pem"
+  cat /usr/local/share/ca-certificates/azcert.crt | docker exec -i pulp bash -c "cat >> /etc/pki/tls/cert.pem"
+  {%- endif %}
+  AZURE_STORAGE_CONNECTION_STRING='DefaultEndpointsProtocol={{ pulp_scheme }};AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint={{ pulp_scheme }}://pulp-azurite:10000/devstoreaccount1;'
+  az storage container create --name pulp-test --connection-string $AZURE_STORAGE_CONNECTION_STRING
+fi
+
 echo ::group::PIP_LIST
 cmd_prefix bash -c "pip3 list && pip3 install pipdeptree && pipdeptree"
 echo ::endgroup::