diff --git a/.github/workflows/update-python-pkg-index.yml b/.github/workflows/update-python-pkg-index.yml new file mode 100644 index 0000000..bd5de99 --- /dev/null +++ b/.github/workflows/update-python-pkg-index.yml @@ -0,0 +1,111 @@ +name: Updates the Python package index upon an event. + +on: + repository_dispatch: + types: [update_package_index] + + # According to the [documentation](https://docs.github.com/en/actions/how-tos/manage-workflow-runs/manually-run-a-workflow#configuring-a-workflow-to-run-manually) + # it is only possible to trigger a workflow manually, if it is located in the default branch. + workflow_dispatch: + inputs: + source_repo: + description: 'Name of the repo that contains the dependency.' + required: true + type: string + source_org: + description: 'Name of the organization/user that owns the dependency repo.' + required: true + type: string + dependency_ref: + description: 'Reference that in the dependency repo that should be checked out and turned into a dependency.' + required: true + type: string + + # We need this until this file is not in `main`, without it the web interface will not pick it up. + # See https://stackoverflow.com/a/71057825 + #pull_request: + +jobs: + update-index: + runs-on: ubuntu-latest + steps: + - name: Print all variables + shell: bash + run: | + echo "source_repo: ${{ inputs.source_repo == '' && github.event.client_payload.source_repo || inputs.source_repo }}" + echo "source_org: ${{ inputs.source_org == '' && github.event.client_payload.source_org || inputs.source_org }}" + echo "dep_ref: ${{ inputs.dependency_ref == '' && github.event.client_payload.dependency_ref || inputs.dependency_ref }}" + echo "payload: ${{ toJson(github.event.client_payload) }}" + + - name: Checkout the `main` branch of the Python package index. + uses: actions/checkout@v4 + with: + path: index_repo + ref: main # We always work on main! + + - name: Checkout the repo of the dependency that should be added to the index. + uses: actions/checkout@v4 + with: + repository: ${{ inputs.source_org == '' && github.event.client_payload.source_org || inputs.source_org }}/${{ inputs.source_repo == '' && github.event.client_payload.source_repo || inputs.source_repo }} + path: ${{ inputs.source_repo == '' && github.event.client_payload.source_repo || inputs.source_repo }} + submodules: 'recursive' + ref: ${{ inputs.dependency_ref == '' && github.event.client_payload.dependency_ref || inputs.dependency_ref }} + + - name: Build the distribution file. + shell: bash + run: | + DEPENDENCY_REPO="${PWD}/${{ inputs.source_repo == '' && github.event.client_payload.source_repo || inputs.source_repo }}" + PACKAGE_BUILD_FOLDER="${PWD}/index_repo/build" + + cd "${DEPENDENCY_REPO}" + python -m pip install build --user + python -m build --wheel --outdir "${PACKAGE_BUILD_FOLDER}" + + - name: Test the distribution file. + shell: bash + run: | + PACKAGE_BUILD_FOLDER="${PWD}/index_repo/build" + DESTINATION_FOLDER="${PWD}/index_repo/${{ inputs.source_repo == '' && github.event.client_payload.source_repo || inputs.source_repo }}" + mkdir -p "${DESTINATION_FOLDER}" + + readarray -t -d "" PACKAGE_FILES < <(find "${PACKAGE_BUILD_FOLDER}" -type f -print0) + for I in ${!PACKAGE_FILES[@]} + do + PACKAGE_FILE="${PACKAGE_FILES[$I]}" + pip install --force-reinstall --upgrade --no-deps "${PACKAGE_FILE}" + if [ $? -ne 0 ] + then + echo "Failed to install package '${PACKAGE_FILE}'" + exit 3 + fi + echo "Successfully tested '${PACKAGE_FILE}'" + cp -t "${DESTINATION_FOLDER}" "${PACKAGE_FILE}" + done + + - name: Rescan the package index and update the static `index.html` files. + shell: bash + env: + CI_COMMIT_MESSAGE: updated dependency "${{ inputs.source_org == '' && github.event.client_payload.source_org || inputs.source_org }}/${{ inputs.source_repo == '' && github.event.client_payload.source_repo || inputs.source_repo }}" + CI_COMMIT_AUTHOR: github-actions[bot] + CI_COMMIT_EMAIL: username@users.noreply.github.com + run: | + cd ./index_repo + + # Not fully sure if this check is useful, because it seems that creating a wheel is not reproducible. + # I.e. creating a wheel from a commit and then generating another wheel will result in a "different", + # in terms of its hash, file than the first time. + if ! git status --porcelain --untracked-files=no ; then + # There are no changed. + echo "There were no changes!" + exit 0 + fi + + # Update all the packages. + python generator.py + + # We directly push to main! + git config --global user.name "${{ env.CI_COMMIT_AUTHOR }}" + git config --global user.email "${{ env.CI_COMMIT_EMAIL }}" + git add . + git commit --no-verify -m "${CI_COMMIT_MESSAGE}" + git push origin main diff --git a/.gitignore b/.gitignore index 493e69b..993a092 100644 --- a/.gitignore +++ b/.gitignore @@ -1,18 +1,2 @@ -# This .gitignore is appropriate for repositories deployed to GitHub Pages and using -# a Gemfile as specified at https://github.com/github/pages-gem#conventional - -# Basic Jekyll gitignores (synchronize to Jekyll.gitignore) -_site/ -.sass-cache/ -.jekyll-cache/ -.jekyll-metadata - -# Additional Ruby/bundler ignore for when you run: bundle install -/vendor - -# Specific ignore for GitHub Pages -# GitHub Pages will always use its own deployed version of pages-gem -# This means GitHub Pages will NOT use your Gemfile.lock and therefore it is -# counterproductive to check this file into the repository. -# Details at https://github.com/github/pages-gem/issues/768 -Gemfile.lock +.token* +build/ diff --git a/README.md b/README.md index 3502756..fc4f940 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,82 @@ -# pypi-index -Python packages index +# Custom GT4Py Python Package Index Server +This repo hosts the custom packages that are needed to use GT4Py, these currently includes: +- [GridTools/dace](https://github.com/GridTools/dace), currently only for the `next`. +- [ghex-org/GHEX](https://github.com/ghex-org/GHEX) + + +# Usage +The repo is intended to work fully automatically, orchestrated by GitHub actions. + +## Workflow `update-python-pkg-index.yml` +This is the main workflow, in short it does: +- Pulls the repo, whose package should be updated. +- Creates a wheel from the repo that has been pulled. +- Tests if the wheel can be installed. +- Updated the package index, i.e. regenerates the `index.html` files, for this `generator.py` is used. +- Creates a commit containing the updated indexes and the generated wheel. +- Pushes the new commit directly to `main`. + +The workflow can be started manually, either through the GitHub web interface or through the `issue_update.sh` script. +In either case some information have to be provided: +- The name of the repo on GitHub, generally referred to as "source repo". +- The owner (user or organization) that owns the repo, generally referred to as "source owner". +- The branch of the repo from which a Python package should be created, generally referred to as "dependency ref". + +> According to the [documentation](https://docs.github.com/en/actions/reference/workflows-and-actions/events-that-trigger-workflows#repository_dispatch) the +> `repository_dispatch` trigger (the one that is used such that _other_ repo can start the update) only works when +> the workflow file is located on the default branch! + + +## `generator.py` +Script for updating the static pages. +It works by scanning subfolders, currently `dace` and `ghex`, and creates an index based on all Python packages it founds in them. +It is usually run by by the workflow automatically. + + +## `issue_update.sh` +A simple script that allows to issue a manual remote update of the index. +For more information please see its help output. + + +## `update_workflows` +This folder contains the workflows that must be installed into the repos containing the dependency, these workflows then triggers the update chain. +Here are the steps that are needed to install them. + + +### Token +The first step is to create an access token for the package index. +It is recommended that a [fine grained access token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens#fine-grained-personal-access-tokens) is used. +The token should only grant access to the index repo and must have the '"Contents" repository permissions (write)' permission. + +Then you must install the token in the depending repo, the updater workflow expect the nae `PKG_UPDATE_TOKEN`. + + +### General Process of Installing the Workflow +The installations of workflows is not straight forward. +First you must activate (uncomment) the `pull_request` trigger and push it. +The net effect is that it will run once and GitHub will pick it up then. +Afterwards you have to disable that trigger again. + + +### DaCe +For DaCe the `dace-updater.yml` must be added to the DaCe repo. +Follow the steps above and place it in its [own dedicated PR](https://github.com/GridTools/dace/pull/12). +Note that it only works if this PR is is included in the `gt4py-next-integration` branch, see [these instructions](https://github.com/GridTools/dace/pull/1). + +The workflow listens for pushes to tags for the form `__gt4py-next-integration_*`, if such a push is detected, it will then inform the index repo about the new version. + + +# Design and Working +The index works currently in "pull mode". +This means that the dependent repos, i.e. DaCe or GHEX, informs the index (this repo), that a new version is available. +The index will then download the depending repo, build the Python package and update the html pages. + +However, it would be conceptually simpler, if the index is passive, i.e. if the dependent repos would build the Python package themself and push it to the index. +This design, "push mode", should become the new operation mode in the future. + + +# TODO: +- Install in DaCe +- Install in GHEX +- Configure the page to use `main` as source. + diff --git a/dace/.gitkeep b/dace/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/generator.py b/generator.py new file mode 100644 index 0000000..9a5090f --- /dev/null +++ b/generator.py @@ -0,0 +1,121 @@ +"""Regenerates the index based on the specified folders. +""" +from typing import Final, Sequence + +import hashlib +import pathlib +import re +import sys + +#: The header for a html page. +#: It contains the opening `
` tag and has the `Titel` interpolation. +HTML_HEADER: Final[str] = """\ + + + +