collection-migration-tests.yml

name: CI/CD

on:
  push:
  pull_request:
  schedule:
  # https://crontab.guru/#53_*/4_*_*_*
  - cron: 53 */4 * * *  # At minute 53 past every 4th hour.

jobs:
  prepare-migration:
    name: pre:lock-core-hash
    runs-on: ubuntu-latest
    env:
      CORE_REPO_SLUG: ansible/ansible

    steps:
    - name: Figure out the last commit hash @ HEAD of ${{ env.CORE_REPO_SLUG }}
      run: >-
        git ls-remote git://github.com/${{ env.CORE_REPO_SLUG }}.git HEAD
        |
        awk '{print$1}'
        >
        ansible-core-ref.lock
    - name: Set CORE_REPO_REF env var
      run: >-
        echo "::set-env name=CORE_REPO_REF::$(cat ansible-core-ref.lock)"
    - name: >-
        Persist the lock file with
        ${{ env.CORE_REPO_SLUG }}@${{ env.CORE_REPO_REF }}
        hash
      uses: actions/upload-artifact@v1
      with:
        name: ansible-core-ref
        path: ansible-core-ref.lock

  migrate-collections:
    name: ${{ matrix.migration-scenario }}:build-migrated
    needs:
    - prepare-migration
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        os:
        - ubuntu-latest
        python-version:
        - 3.7
        migration-scenario:
        - base
    env:
      CORE_REPO_SLUG: ansible/ansible

    steps:
    - uses: actions/download-artifact@v1
      with:
        name: ansible-core-ref
        path: .
    - name: Set CORE_REPO_REF env var
      run: >-
        echo "::set-env name=CORE_REPO_REF::$(cat ansible-core-ref.lock)"
    - name: Check out the src
      uses: actions/checkout@master
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v1
      with:
        python-version: ${{ matrix.python-version }}
    - name: Restore pip cache
      uses: actions/cache@v1
      with:
        path: ~/.cache/pip
        key: ${{ runner.os }}-pip-${{ hashFiles('requirements.in') }}-${{ hashFiles('requirements.txt') }}
        restore-keys: |
          ${{ runner.os }}-pip-
          ${{ runner.os }}-
    - name: Uninstall previously installed Ansible via Apt
      run: sudo apt remove --yes ansible
    - name: Uninstall previously installed Ansible via Pip
      run: python -m pip uninstall ansible
    - name: install binary deps
      run: sudo apt install parallel
    - name: Install migration script deps
      run: python -m pip install -r requirements.in -c requirements.txt
    - name: Install Ansible==${{ env.CORE_REPO_REF }}
      run: python -m pip install git+https://github.com/ansible/ansible.git@${{ env.CORE_REPO_REF }}
    - name: Configure user settings in Git
      run: |
        git config --global user.email "info@ansible.com"
        git config --global user.name "Ansible Core Team"
    - name: Run migration scenario ${{ matrix.migration-scenario }}
      run: >-
        python -m
        migrate
        -m
        -s "scenarios/${{ matrix.migration-scenario }}"
        --refresh "${{ env.CORE_REPO_REF }}"
        --skip-publish
    - name: >-
        Create the tarballs dir for migrated Ansible Core
        and collections
      run: mkdir -pv .cache/migrated-tarballs
    - name: >-
        Generate tarballs out of the migrated Ansible Core
        as per ${{ matrix.migration-scenario }}
      run: >-
          tar czvf
          ".cache/migrated-tarballs/ansible-${{ matrix.migration-scenario }}.tar.gz"
          .cache/releases/devel.git
    - name: >-
        Generate tarballs out of the migrated Ansible Collections
        Git repositories as per ${{ matrix.migration-scenario }}
      run: >-
          tar czvf
          ".cache/migrated-tarballs/ansible-collections-${{ matrix.migration-scenario }}.tar.gz"
          .cache/collections/ansible_collections
    - name: >-
        Store the migrated Ansible Core
        as per ${{ matrix.migration-scenario }}
      uses: actions/upload-artifact@v1
      with:
        name: >-
          migrated-core-${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.migration-scenario }}
        path: .cache/migrated-tarballs
    - name: >-
        Generate tarballs out of migrated collections, one by one
      run: |
          for coll_dir in `ls .cache/collections/*/*/* -d`
          do
              (
                  echo "Generating an artifact for ${coll_dir}..."
                  cd "${coll_dir}"
                  pwd
                  ansible-galaxy collection build --output-path "${GITHUB_WORKSPACE}/.cache/collection-tarballs"
              )
          done
    - name: Store migrated collection artifacts
      uses: actions/upload-artifact@v1
      with:
        name: >-
          migrated-collections-${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.migration-scenario }}
        path: .cache/collection-tarballs

  publish-migrated-core:
    name: ${{ matrix.migration-scenario }}:publish:core
    needs:
    - migrate-collections
    if: >-
      github.event_name != 'pull_request'
      && github.repository == 'ansible-community/collection_migration'
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        os:
        - ubuntu-latest
        python-version:
        - 3.7
        migration-scenario:
        - base

    steps:
    - uses: actions/checkout@master
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v1
      with:
        python-version: ${{ matrix.python-version }}
    - name: Restore pip cache
      uses: actions/cache@v1
      with:
        path: ~/.cache/pip
        key: ${{ runner.os }}-pip-${{ hashFiles('requirements.in') }}-${{ hashFiles('requirements.txt') }}
        restore-keys: |
          ${{ runner.os }}-pip-
          ${{ runner.os }}-
    - name: Uninstall previously installed Ansible via Apt
      run: sudo apt remove --yes ansible
    - name: Uninstall previously installed Ansible via Pip
      run: python -m pip uninstall ansible
    - name: Install migration script deps
      run: python -m pip install -r requirements.in -c requirements.txt
    - name: >-
        Download migrated Ansible Core and collections for
        ${{ matrix.migration-scenario }} scenario
      uses: actions/download-artifact@v1
      with:
        name: >-
          migrated-core-${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.migration-scenario }}
        path: .cache/migrated-tarballs
    - name: >-
        Retrieve the migrated Ansible Core for
        ${{ matrix.migration-scenario }} scenario
        out of the tarball artifacts
      run: >-
          tar xzvfh
          ".cache/migrated-tarballs/ansible-${{ matrix.migration-scenario }}.tar.gz"
          .cache/releases/devel.git
    - name: >-
        Install Ansible from the migrated Core for
        ${{ matrix.migration-scenario }} scenario
      run: >-
        python -m
        pip install
        .cache/releases/devel.git
    - name: >-
        Help output of the migrate Ansible Core for
        ${{ matrix.migration-scenario }} scenario
      run: >-
        ansible --help
    - name: >-
        Version output of the migrate Ansible Core for
        ${{ matrix.migration-scenario }} scenario
      run: >-
        ansible --version
    - name: Prepare user's SSH config dir
      run: |
        mkdir -p ~/.ssh
        chmod u=rwx,g=,o= ~/.ssh
    - name: Ensure GitHub's public SSH keys are in known hosts
      run: >-
        cat resources/github_known_hosts.tmpl
        >>
        ~/.ssh/known_hosts
    - name: >-
        Auto-publish post-migration Ansible Core version
        to GitHub repo as per ${{ matrix.migration-scenario }}
      env:
        GITHUB_APP_IDENTIFIER: 41435
        GITHUB_PRIVATE_KEY: ${{ secrets.GITHUB_PRIVATE_KEY }}
      run: >-
        python -m
        migrate
        -s "scenarios/${{ matrix.migration-scenario }}"
        --target-github-org ansible-collection-migration
        --skip-migration
        --push-migrated-core

  publish-migrated-collections:
    name: ${{ matrix.migration-scenario }}:publish:collections
    needs:
    - migrate-collections
    if: >-
      github.event_name != 'pull_request'
      && github.repository == 'ansible-community/collection_migration'
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        os:
        - ubuntu-latest
        python-version:
        - 3.7
        migration-scenario:
        - base

    steps:
    - uses: actions/checkout@master
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v1
      with:
        python-version: ${{ matrix.python-version }}
    - name: Restore pip cache
      uses: actions/cache@v1
      with:
        path: ~/.cache/pip
        key: ${{ runner.os }}-pip-${{ hashFiles('requirements.in') }}-${{ hashFiles('requirements.txt') }}
        restore-keys: |
          ${{ runner.os }}-pip-
          ${{ runner.os }}-
    - name: Uninstall previously installed Ansible via Apt
      run: sudo apt remove --yes ansible
    - name: Uninstall previously installed Ansible via Pip
      run: python -m pip uninstall ansible
    - name: Install migration script deps
      run: python -m pip install -r requirements.in -c requirements.txt
    - name: >-
        Download migrated Ansible Core and collections for
        ${{ matrix.migration-scenario }} scenario
      uses: actions/download-artifact@v1
      with:
        name: >-
          migrated-core-${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.migration-scenario }}
        path: .cache/migrated-tarballs
    - name: >-
        Retrieve the migrated Ansible Core for
        ${{ matrix.migration-scenario }} scenario
        out of the tarball artifacts
      run: >-
          tar xzvfh
          ".cache/migrated-tarballs/ansible-${{ matrix.migration-scenario }}.tar.gz"
          .cache/releases/devel.git
    - name: >-
        Retrieve the migrated Ansible Collection Git dirs for
        ${{ matrix.migration-scenario }} scenario
        out of the tarball artifacts
      run: >-
          tar xzvfh
          ".cache/migrated-tarballs/ansible-collections-${{ matrix.migration-scenario }}.tar.gz"
          .cache/collections/ansible_collections
    - name: >-
        Install Ansible from the migrated Core for
        ${{ matrix.migration-scenario }} scenario
      run: >-
        python -m
        pip install
        .cache/releases/devel.git
    - name: Download migrated collection artifacts
      uses: actions/download-artifact@v1
      with:
        name: >-
          migrated-collections-${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.migration-scenario }}
        path: .cache/collection-tarballs
    - name: >-
        Install the migrated collections
      run: |
          ansible-galaxy collection install .cache/collection-tarballs/*.tar.gz
    - name: >-
        List the migrated collections
      run: |
          ls -Ral ~/.ansible/collections/
    - name: Prepare user's SSH config dir
      run: |
        mkdir -p ~/.ssh
        chmod u=rwx,g=,o= ~/.ssh
    - name: Ensure GitHub's public SSH keys are in known hosts
      run: >-
        cat resources/github_known_hosts.tmpl
        >>
        ~/.ssh/known_hosts
    - name: >-
        Auto-publish the migrated collection repos
        to GitHub repos as per ${{ matrix.migration-scenario }}
      env:
        GITHUB_APP_IDENTIFIER: 41435
        GITHUB_PRIVATE_KEY: ${{ secrets.GITHUB_PRIVATE_KEY }}
      run: >-
        python -m
        migrate
        -s "scenarios/${{ matrix.migration-scenario }}"
        --target-github-org ansible-collection-migration
        --skip-migration
        --publish-to-github

  sanity-test-collections:
    name: ${{ matrix.migration-scenario }}:tests:sanity
    needs:
    - migrate-collections
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        os:
        - ubuntu-latest
        python-version:
        - 3.7
        migration-scenario:
        - base
    steps:
    - uses: actions/checkout@master
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v1
      with:
        python-version: ${{ matrix.python-version }}
    - name: Restore pip cache
      uses: actions/cache@v1
      with:
        path: ~/.cache/pip
        key: ${{ runner.os }}-pip-${{ hashFiles('requirements.in') }}-${{ hashFiles('requirements.txt') }}
        restore-keys: |
          ${{ runner.os }}-pip-
          ${{ runner.os }}-
    - name: Uninstall previously installed Ansible via Apt
      run: sudo apt remove --yes ansible
    - name: Uninstall previously installed Ansible via Pip
      run: python -m pip uninstall ansible
    - name: >-
        Download migrated Ansible Core and collections for
        ${{ matrix.migration-scenario }} scenario
      uses: actions/download-artifact@v1
      with:
        name: >-
          migrated-core-${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.migration-scenario }}
        path: .cache/migrated-tarballs
    - name: >-
        Retrieve the migrated Ansible Core for
        ${{ matrix.migration-scenario }} scenario
        out of the tarball artifacts
      run: >-
          tar xzvfh
          ".cache/migrated-tarballs/ansible-${{ matrix.migration-scenario }}.tar.gz"
          .cache/releases/devel.git
    - name: >-
        Install Ansible from the migrated Core for
        ${{ matrix.migration-scenario }} scenario
      run: >-
        python -m
        pip install
        .cache/releases/devel.git
    - name: Download migrated collection artifacts
      uses: actions/download-artifact@v1
      with:
        name: >-
          migrated-collections-${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.migration-scenario }}
        path: .cache/collection-tarballs
    - name: >-
        Install the migrated collections
      run: |
          ansible-galaxy collection install .cache/collection-tarballs/*.tar.gz
    - name: >-
        [NON-FAILING]
        Run sanity tests against all migrated collections, one by one
      run: |
          for coll_dir in `ls ~/.ansible/collections/*/*/* -d`
          do
              (
                  echo "Sanity-testing ${coll_dir}..."
                  cd "${coll_dir}"
                  pwd
                  ansible-test sanity --requirements --python "${{ matrix.python-version }}" -vvv || :
              )
          done

  unit-test-collections:
    name: ${{ matrix.migration-scenario }}:tests:units
    needs:
    - migrate-collections
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        os:
        - ubuntu-latest
        python-version:
        - 3.7
        migration-scenario:
        - base
    steps:
    - uses: actions/checkout@master
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v1
      with:
        python-version: ${{ matrix.python-version }}
    - name: Restore pip cache
      uses: actions/cache@v1
      with:
        path: ~/.cache/pip
        key: ${{ runner.os }}-pip-${{ hashFiles('requirements.in') }}-${{ hashFiles('requirements.txt') }}
        restore-keys: |
          ${{ runner.os }}-pip-
          ${{ runner.os }}-
    - name: Uninstall previously installed Ansible via Apt
      run: sudo apt remove --yes ansible
    - name: Uninstall previously installed Ansible via Pip
      run: python -m pip uninstall ansible
    - name: >-
        Download migrated Ansible Core and collections for
        ${{ matrix.migration-scenario }} scenario
      uses: actions/download-artifact@v1
      with:
        name: >-
          migrated-core-${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.migration-scenario }}
        path: .cache/migrated-tarballs
    - name: >-
        Retrieve the migrated Ansible Core for
        ${{ matrix.migration-scenario }} scenario
        out of the tarball artifacts
      run: >-
          tar xzvfh
          ".cache/migrated-tarballs/ansible-${{ matrix.migration-scenario }}.tar.gz"
          .cache/releases/devel.git
    - name: >-
        Install Ansible from the migrated Core for
        ${{ matrix.migration-scenario }} scenario
      run: >-
        python -m
        pip install
        .cache/releases/devel.git
    - name: Download migrated collection artifacts
      uses: actions/download-artifact@v1
      with:
        name: >-
          migrated-collections-${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.migration-scenario }}
        path: .cache/collection-tarballs
    - name: >-
        Install the migrated collections
      run: |
          ansible-galaxy collection install .cache/collection-tarballs/*.tar.gz
    - name: >-
        [NON-FAILING]
        Run unit tests against all migrated collections, one by one
      run: |
          for coll_dir in `ls ~/.ansible/collections/*/*/* -d`
          do
              (
                  echo "Unit-testing ${coll_dir}..."
                  cd "${coll_dir}"
                  pwd
                  [[ ! -d "tests/unit" ]] && echo This collection does not have unit tests. Skipping... && continue
                  ansible-test units --requirements --python "${{ matrix.python-version }}" -vvv || :
              )
          done