Skip to content
This repository has been archived by the owner on Apr 9, 2022. It is now read-only.

Commit

Permalink
Browse files Browse the repository at this point in the history
improve CI (#26)
* update GitHub Actions

* check setup.py as well

* add build, test, and publish jobs

* fix
  • Loading branch information
epwalsh committed Aug 20, 2020
1 parent 28b7cf5 commit 6d0e63e
Show file tree
Hide file tree
Showing 5 changed files with 289 additions and 40 deletions.
167 changes: 156 additions & 11 deletions .github/workflows/main.yml
@@ -1,35 +1,180 @@
name: CI

on:
pull_request:
branches:
- master
push:
branches:
- master
release:
types: [published]

jobs:
build:
checks:
name: Checks
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v1

- name: Setup Python
uses: actions/setup-python@v1
with:
python-version: 3.7

- uses: actions/cache@v2
with:
path: ${{ env.pythonLocation }}
key: ${{ runner.os }}-pydeps-${{ env.pythonLocation }}-${{ hashFiles('requirements.txt') }}-${{ hashFiles('dev-requirements.txt') }}

- name: Install requirements
run: |
pip install -r requirements.txt
pip install pytest pytest-cov flake8 black mypy
pip install --upgrade pip setuptools wheel
pip install --upgrade --upgrade-strategy eager -r requirements.txt
pip install --upgrade --upgrade-strategy eager -r dev-requirements.txt
- name: Show pip freeze
run: |
pip freeze
- name: Check setup.py
if: always()
run: |
python setup.py check
- name: Format
if: always()
run: |
black --check .
- name: Lint
if: always()
run: |
flake8
- name: Type check
if: always()
run: |
mypy allennlp_semparse tests --ignore-missing-imports --no-strict-optional --no-site-packages
- name: Run tests
if: always()
run: |
pytest --cov=allennlp_semparse/ --cov-report=xml
pytest -v --color=yes --cov=allennlp_semparse/ --cov-report=xml
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v1.0.2
if: github.repository == 'allenai/allennlp-semparse'
uses: codecov/codecov-action@v1
with:
token: 72a3fbab-4667-4dc3-b309-86200f868d33
- name: Lint
file: ./coverage.xml
# Ignore codecov failures as the codecov server is not
# very reliable but we don't want to report a failure
# in the github UI just because the coverage report failed to
# be published.
fail_ci_if_error: false

build_package:
name: Build Package
runs-on: ubuntu-latest

steps:
- uses: actions/checkout@v2

- name: Setup Python
uses: actions/setup-python@v1
with:
python-version: 3.7

- uses: actions/cache@v2
with:
path: ${{ env.pythonLocation }}
key: ${{ runner.os }}-pydeps-${{ env.pythonLocation }}-${{ hashFiles('requirements.txt') }}-${{ hashFiles('dev-requirements.txt') }}

- name: Install requirements
run: |
flake8 -v
black -v --check .
- name: Type check
pip install --upgrade pip setuptools wheel
pip install --upgrade --upgrade-strategy eager -r requirements.txt
pip install --upgrade --upgrade-strategy eager -r dev-requirements.txt
- name: Check version and release tag match
if: github.event_name == 'release'
run: |
# Remove 'refs/tags/' to get the actual tag from the release.
TAG=${GITHUB_REF#refs/tags/};
VERSION=$(scripts/get_version.py current)
if [ "$TAG" != "$VERSION" ]; then
echo "Bad tag or version. Tag $TAG does not match $VERSION";
exit 1;
fi
- name: Build Package
run: |
python setup.py bdist_wheel sdist
- name: Save package
uses: actions/upload-artifact@v1
with:
name: semparse-package
path: dist

test_package:
name: Test Package
needs: [build_package] # needs the package artifact created from 'build_package' job.
runs-on: ubuntu-latest

steps:
- uses: actions/checkout@v2

- name: Cleanup directory
run: |
# Remove the source code so that it doesn't conflict with the wheel
# installation.
rm -rf allennlp_semparse/
- name: Setup Python
uses: actions/setup-python@v1
with:
python-version: 3.7

- name: Install requirements
run: |
pip install --upgrade pip setuptools wheel
- name: Download semparse package
uses: actions/download-artifact@v1
with:
name: semparse-package
path: dist

- name: Install semparse package
run: |
pip install $(ls dist/*.whl)
publish:
name: Publish to PyPI
if: github.event_name == 'release'
needs: [checks, build_package, test_package]
runs-on: ubuntu-latest

steps:
- uses: actions/checkout@v2

- name: Setup Python
uses: actions/setup-python@v1
with:
python-version: 3.7

- name: Install requirements
run: |
mypy allennlp_semparse --ignore-missing-imports --no-strict-optional --no-site-packages
pip install --upgrade pip setuptools wheel twine
- name: Download semparse package
uses: actions/download-artifact@v1
with:
name: semparse-package
path: dist

- name: Upload to PyPI
env:
PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
run: twine upload -u allennlp -p $PYPI_PASSWORD dist/*
2 changes: 1 addition & 1 deletion allennlp_semparse/version.py
@@ -1,6 +1,6 @@
_MAJOR = "0"
_MINOR = "0"
_REVISION = "1-unreleased"
_REVISION = "1rc1"

VERSION_SHORT = "{0}.{1}".format(_MAJOR, _MINOR)
VERSION = "{0}.{1}.{2}".format(_MAJOR, _MINOR, _REVISION)
36 changes: 36 additions & 0 deletions dev-requirements.txt
@@ -0,0 +1,36 @@
#### TESTING-RELATED PACKAGES ####

# Checks style, syntax, and other useful errors.
flake8

# Static type checking
mypy==0.782

# Automatic code formatting
black

# Running unit tests.
pytest

# Allows generation of coverage reports with pytest.
pytest-cov

# Lets you run tests in forked subprocesses.
pytest-forked

# Lets you run tests in parallel.
pytest-xdist

# Allows codecov to generate coverage reports
coverage
codecov

# For running tests that aren't 100% reliable
flaky

#### PACKAGE-UPLOAD PACKAGES ####

# Pypi uploads
twine>=1.11.0
setuptools
wheel
56 changes: 56 additions & 0 deletions scripts/get_version.py
@@ -0,0 +1,56 @@
#!/usr/bin/env python3

import argparse
from typing import Dict


def parse_args():
parser = argparse.ArgumentParser()
parser.add_argument("version_type", choices=["stable", "latest", "current"])
parser.add_argument("--minimal", action="store_true", default=False)
return parser.parse_args()


def post_process(version: str, minimal: bool = False):
if version.startswith("v"):
return version if not minimal else version[1:]
return version if minimal else f"v{version}"


def get_current_version() -> str:
VERSION: Dict[str, str] = {}
with open("allennlp_semparse/version.py", "r") as version_file:
exec(version_file.read(), VERSION)
return VERSION["VERSION"]


def get_latest_version() -> str:
# Import this here so this requirements isn't mandatory when we just want to
# call `get_current_version`.
import requests

resp = requests.get("https://api.github.com/repos/allenai/allennlp-semparse/tags")
return resp.json()[0]["name"]


def get_stable_version() -> str:
import requests

resp = requests.get("https://api.github.com/repos/allenai/allennlp-semparse/releases/latest")
return resp.json()["tag_name"]


def main() -> None:
opts = parse_args()
if opts.version_type == "stable":
print(post_process(get_stable_version(), opts.minimal))
elif opts.version_type == "latest":
print(post_process(get_latest_version(), opts.minimal))
elif opts.version_type == "current":
print(post_process(get_current_version(), opts.minimal))
else:
raise NotImplementedError


if __name__ == "__main__":
main()
68 changes: 40 additions & 28 deletions setup.py
@@ -1,5 +1,4 @@
from setuptools import setup, find_packages
import sys
import os

# PEP0440 compatible formatted version, see:
Expand All @@ -24,33 +23,48 @@
# Load requirements.txt with a special case for allennlp so we can handle
# cross-library integration testing.
with open("requirements.txt") as requirements_file:
install_requirements = requirements_file.readlines()
install_requirements = [
r for r in install_requirements if "git+git://github.com/allenai/allennlp" not in r
]
if not os.environ.get("EXCLUDE_ALLENNLP_IN_SETUP"):
# Warning: This will not give you the desired version if you've already
# installed allennlp! See https://github.com/pypa/pip/issues/5898.
#
# There used to be an alternative to this using `dependency_links`
# (https://stackoverflow.com/questions/3472430), but pip decided to
# remove this in version 19 breaking numerous projects in the process.
# See https://github.com/pypa/pip/issues/6162.
#
# As a mitigation, run `pip uninstall allennlp` before installing this
# package.
sha = "28028a0544632307fe48e66a15a4b2a22c3dddda"
requirement = f"allennlp @ git+ssh://git@github.com/allenai/allennlp@{sha}#egg=allennlp"
install_requirements.append(requirement)
import re

# make pytest-runner a conditional requirement,
# per: https://github.com/pytest-dev/pytest-runner#considerations
needs_pytest = {"pytest", "test", "ptr"}.intersection(sys.argv)
pytest_runner = ["pytest-runner"] if needs_pytest else []
def requirement_is_allennlp(req: str) -> bool:
if req == "allennlp":
return True
if re.match(r"^allennlp[>=<]", req):
return True
if re.match(r"^(git\+)?(https|ssh)://(git@)?github\.com/.*/allennlp\.git", req):
return True
return False

setup_requirements = [
# add other setup requirements as necessary
] + pytest_runner
def fix_url_dependencies(req: str) -> str:
"""Pip and setuptools disagree about how URL dependencies should be handled."""
m = re.match(
r"^(git\+)?(https|ssh)://(git@)?github\.com/([\w-]+)/(?P<name>[\w-]+)\.git", req
)
if m is None:
return req
else:
return f"{m.group('name')} @ {req}"

install_requirements = []
allennlp_requirements = []
for line in requirements_file:
line = line.strip()
if line.startswith("#") or len(line) <= 0:
continue
if requirement_is_allennlp(line):
allennlp_requirements.append(line)
else:
install_requirements.append(line)

assert len(allennlp_requirements) == 1
allennlp_override = os.environ.get("ALLENNLP_VERSION_OVERRIDE")
if allennlp_override is not None:
if len(allennlp_override) > 0:
allennlp_requirements = [allennlp_override]
else:
allennlp_requirements = []

install_requirements.extend(allennlp_requirements)
install_requirements = [fix_url_dependencies(req) for req in install_requirements]

setup(
name="allennlp_semparse",
Expand All @@ -75,8 +89,6 @@
license="Apache",
packages=find_packages(exclude=["*.tests", "*.tests.*", "tests.*", "tests"]),
install_requires=install_requirements,
setup_requires=setup_requirements,
tests_require=["pytest", "flaky", "responses>=0.7"],
include_package_data=True,
python_requires=">=3.6.1",
zip_safe=False,
Expand Down

0 comments on commit 6d0e63e

Please sign in to comment.