Skip to content
Permalink
Browse files

CI: ratchet up mypy

  • Loading branch information
grahamc committed Mar 12, 2020
1 parent 8262e7d commit 28b84043c1e2b59ce3a3de52fe0ba6b237ab2920
Showing with 111 additions and 3 deletions.
  1. +1 −1 .github/workflows/ci.yml
  2. +16 −2 ci.nix
  3. +38 −0 ci/mypy-ratchet.sh
  4. +56 −0 ci/ratchet.py
@@ -1 +1 @@
{"jobs":{"black":{"runs-on":"ubuntu-latest","steps":[{"name":"Checkout","uses":"actions/checkout@v2"},{"name":"Nix","uses":"cachix/install-nix-action@v7"},{"name":"Black","run":"nix-shell --run \"black . --check --diff\""}]},"build":{"runs-on":"ubuntu-latest","steps":[{"name":"Checkout","uses":"actions/checkout@v2"},{"name":"Nix","uses":"cachix/install-nix-action@v7"},{"name":"Build","run":"nix-build --quiet release.nix -A build.x86_64-linux -I nixpkgs=channel:19.09"}]},"ciCheck":{"runs-on":"ubuntu-latest","steps":[{"name":"Checkout","uses":"actions/checkout@v2"},{"name":"Nix","uses":"cachix/install-nix-action@v7"},{"name":"Check CI","run":"cp ./.github/workflows/ci.yml /tmp/ci.reference.yml\nnix-build ci.nix --no-out-link | bash\ndiff ./.github/workflows/ci.yml /tmp/ci.reference.yml || exit 1\n"}]},"coverage":{"runs-on":"ubuntu-latest","steps":[{"name":"Checkout","uses":"actions/checkout@v2"},{"name":"Nix","uses":"cachix/install-nix-action@v7"},{"name":"Coverage","run":"nix-shell --exclude tarball --run \"./coverage-tests.py -a '!libvirtd,!gce,!ec2,!azure' -v\""}]},"mypy":{"runs-on":"ubuntu-latest","steps":[{"name":"Checkout","uses":"actions/checkout@v2"},{"name":"Nix","uses":"cachix/install-nix-action@v7"},{"name":"MyPy","run":"nix-shell --run \"mypy nixops\""}]},"parsing":{"runs-on":"ubuntu-latest","steps":[{"name":"Checkout","uses":"actions/checkout@v2"},{"name":"Nix","uses":"cachix/install-nix-action@v7"},{"name":"Parsing","run":"find . -name \"*.nix\" -exec nix-instantiate --parse --quiet {} >/dev/null +"}]}},"name":"CI","on":{"push":{"branches":["*"]}}}
{"jobs":{"black":{"runs-on":"ubuntu-latest","steps":[{"name":"Checkout","uses":"actions/checkout@v2"},{"name":"Nix","uses":"cachix/install-nix-action@v7"},{"name":"Black","run":"nix-shell --run \"black . --check --diff\""}]},"build":{"runs-on":"ubuntu-latest","steps":[{"name":"Checkout","uses":"actions/checkout@v2"},{"name":"Nix","uses":"cachix/install-nix-action@v7"},{"name":"Build","run":"nix-build --quiet release.nix -A build.x86_64-linux -I nixpkgs=channel:19.09"}]},"ciCheck":{"runs-on":"ubuntu-latest","steps":[{"name":"Checkout","uses":"actions/checkout@v2"},{"name":"Nix","uses":"cachix/install-nix-action@v7"},{"name":"Check CI","run":"cp ./.github/workflows/ci.yml /tmp/ci.reference.yml\nnix-build ci.nix --no-out-link | bash\ndiff ./.github/workflows/ci.yml /tmp/ci.reference.yml || exit 1\n"}]},"coverage":{"runs-on":"ubuntu-latest","steps":[{"name":"Checkout","uses":"actions/checkout@v2"},{"name":"Nix","uses":"cachix/install-nix-action@v7"},{"name":"Coverage","run":"nix-shell --exclude tarball --run \"./coverage-tests.py -a '!libvirtd,!gce,!ec2,!azure' -v\""}]},"mypy":{"runs-on":"ubuntu-latest","steps":[{"name":"Checkout","uses":"actions/checkout@v2"},{"name":"Nix","uses":"cachix/install-nix-action@v7"},{"name":"MyPy","run":"nix-shell --run \"mypy nixops\""}]},"mypy-ratchet":{"runs-on":"ubuntu-latest","steps":[{"name":"Checkout","uses":"actions/checkout@v2"},{"name":"Nix","uses":"cachix/install-nix-action@v7"},{"name":"setup-shell","run":"nix-shell --run \"true\""},{"name":"mypy-ratchet","run":"./ci/mypy-ratchet.sh"}]},"parsing":{"runs-on":"ubuntu-latest","steps":[{"name":"Checkout","uses":"actions/checkout@v2"},{"name":"Nix","uses":"cachix/install-nix-action@v7"},{"name":"Parsing","run":"find . -name \"*.nix\" -exec nix-instantiate --parse --quiet {} >/dev/null +"}]}},"name":"CI","on":{"pull_request":{"branches":["*"]},"push":{"branches":["*"]}}}
18 ci.nix
@@ -18,11 +18,25 @@ let
};
ci = {
on.push.branches = [ "*" ];
on.pull_request.branches = [ "*" ];

name = "CI";
jobs = {

mypy-ratchet = mkJob [
{
name = "setup-shell";
run = ''nix-shell --run "true"'';
}
{
name = "mypy-ratchet";
run = "./ci/mypy-ratchet.sh";
}
];

parsing = mkJob [{
name = "Parsing";
run = "find . -name \"*.nix\" -exec nix-instantiate --parse --quiet {} >/dev/null +";
name = "Parsing";
run = "find . -name \"*.nix\" -exec nix-instantiate --parse --quiet {} >/dev/null +";
}];
mypy = mkJob [{
name = "MyPy";
@@ -0,0 +1,38 @@
#!/usr/bin/env nix-shell
#!nix-shell -i bash ../shell.nix

set -eu

scratch=$(mktemp -d -t tmp.XXXXXXXXXX)
function finish {
rm -rf "$scratch"
}
trap finish EXIT

head=$(git rev-parse HEAD)
base=origin/${GITHUB_BASE_REF:-master}

git fetch origin

echo "Checking base branch at %s, then PR at %s...\n" "$base" "$head"

git checkout "$base"
mypy \
--any-exprs-report "$scratch/base" \
--linecount-report "$scratch/base" \
--lineprecision-report "$scratch/base" \
--txt-report "$scratch/base" \
nixops

git checkout "$head"
mypy \
--any-exprs-report "$scratch/head" \
--linecount-report "$scratch/head" \
--lineprecision-report "$scratch/head" \
--txt-report "$scratch/head" \
nixops

diff --ignore-all-space -u100 -r "$scratch/base/" "$scratch/head/" || true

mypy ./ci/ratchet.py
python3 ./ci/ratchet.py "$scratch"
@@ -0,0 +1,56 @@
import sys
import re
from typing import Dict, Tuple, Optional, List, TextIO

from pprint import pprint

report: Dict[str, List[Optional[float]]] = {}

extract_line = re.compile(
"^\|\s+(?P<module>[^\s]*)\s+\|\s+(?P<percent>\d\d?\.\d\d)% imprecise \|"
)


def load(idx: int, fp: TextIO):
global report
for line in fp:
r = extract_line.match(line)
if r:
module: str = r.group("module")
percent: float = float(r.group("percent"))

if module not in report:
report[module] = [None, None]
report[module][idx] = percent


with open(f"{sys.argv[1]}/base/index.txt") as fp:
load(0, fp)
with open(f"{sys.argv[1]}/head/index.txt") as fp:
load(1, fp)

good: List[Tuple[str, float, float]] = []
bad: List[Tuple[str, float, float]] = []

for module, values in report.items():
base: Optional[float] = values[0]
head: Optional[float] = values[1]

if base is not None and head is not None:
if base > head:
good.append((module, base, head))
elif base < head:
bad.append((module, base, head))

if len(bad) > 0:
print("Imprecision went up:")
for module, base, head in bad:
print(f"{module}:\t\t{base} -> {head}")

if len(good) > 0:
print("Imprecision went down:")
for module, base, head in good:
print(f"{module}:\t\t{base} -> {head}")

if len(bad) > 0:
sys.exit(1)

0 comments on commit 28b8404

Please sign in to comment.
You can’t perform that action at this time.