From f3d1da02f433f2a95a314f20c548f7a7e78c720d Mon Sep 17 00:00:00 2001 From: Romain Marcadier-Muller Date: Wed, 14 Aug 2019 14:07:05 -0700 Subject: [PATCH] feat(python): check distribution artifacts with `twine` (#711) Adds a `twine check` invocation at the end of building the distribution artifacts for PyPI, in order to ensure the produced bundles will render propertly on PyPI, and be valid artifacts. This particularly guards against "tool old" versions of `setuptools`, but could also catch certain other issues. Fixes #710 --- CONTRIBUTING.md | 29 ++++++---- packages/jsii-pacmak/lib/logging.ts | 18 ++++-- packages/jsii-pacmak/lib/targets/python.ts | 17 ++++++ superchain/Dockerfile | 64 ++++++++++------------ 4 files changed, 76 insertions(+), 52 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b1a069cc0e..0d212d9fd2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -11,20 +11,25 @@ Due to the polyglot nature of `jsii`, the toolchain requirements are somewhat more complicated than for most projects. In order to locally develop `jsii`, you will need the following tools: -- [Node 8.11.0] or later -- An OpenJDK-8 distribution - + [Oracle's OpenJDK8] - + [Amazon Corretto 8] -- [.NET Core 2.0] or later -- [Python 3.6.5] or later -- [Ruby 2.5.1] or later - -[Node 8.11.0]: https://nodejs.org/download/release/v8.11.0/ +- [Node `8.11.0`] or later +- An OpenJDK-8 distribution (e.g: [Oracle's OpenJDK8], [Amazon Corretto 8]) + + [`maven >= 3.0.5`](https://maven.apache.org) +- [.NET Core `2.0`] or later + + *Recommended:* [`mono >= 5`](https://www.mono-project.com) +- [Python `3.6.5`] or later + + [`pip`](https://pip.pypa.io/en/stable/installing/) + + [`setuptools >= 38.6.0`](https://pypi.org/project/setuptools/) + + [`wheel`](https://pypi.org/project/wheel/) + + *Recommended:* [`twine`](https://pypi.org/project/twine/) +- [Ruby `2.4.4p296`] or later + + [`bundler ~> 1.17.2`](https://bundler.io) + +[Node `8.11.0`]: https://nodejs.org/download/release/v8.11.0/ [Oracle's OpenJDK8]: http://openjdk.java.net/install/ [Amazon Corretto 8]: https://aws.amazon.com/corretto/ -[.NET Core 2.0]: https://www.microsoft.com/net/download -[Python 3.6.5]: https://www.python.org/downloads/release/python-365/ -[Ruby 2.5.1]: https://www.ruby-lang.org/en/news/2018/03/28/ruby-2-5-1-released/ +[.NET Core `2.0`]: https://www.microsoft.com/net/download +[Python `3.6.5`]: https://www.python.org/downloads/release/python-365/ +[Ruby `2.4.4p296`]: https://www.ruby-lang.org/en/news/2018/03/28/ruby-2-5-1-released/ ### Alterative: build in Docker diff --git a/packages/jsii-pacmak/lib/logging.ts b/packages/jsii-pacmak/lib/logging.ts index 16c20826dd..50a923415a 100644 --- a/packages/jsii-pacmak/lib/logging.ts +++ b/packages/jsii-pacmak/lib/logging.ts @@ -1,14 +1,20 @@ -export let level = 0; - -export const LEVEL_INFO = 1; -export const LEVEL_VERBOSE = 2; - export enum Level { + WARN = -1, QUIET = 0, INFO = 1, VERBOSE = 2, } +export const LEVEL_INFO: number = Level.INFO; +export const LEVEL_VERBOSE: number = Level.VERBOSE; + +/** The minimal logging level for messages to be emitted. */ +export let level = Level.QUIET; + +export function warn(fmt: string, ...args: any[]) { + log(Level.WARN, fmt, ...args); +} + export function info(fmt: string, ...args: any[]) { log(Level.INFO, fmt, ...args); } @@ -23,4 +29,4 @@ function log(messageLevel: Level, fmt: string, ...args: any[]) { // tslint:disable-next-line:no-console console.error.call(console, ...[ `[jsii-pacmak] [${levelName}]`, fmt, ...args ]); } -} \ No newline at end of file +} diff --git a/packages/jsii-pacmak/lib/targets/python.ts b/packages/jsii-pacmak/lib/targets/python.ts index cd0b262943..a163135202 100644 --- a/packages/jsii-pacmak/lib/targets/python.ts +++ b/packages/jsii-pacmak/lib/targets/python.ts @@ -6,6 +6,7 @@ import * as reflect from 'jsii-reflect'; import * as spec from 'jsii-spec'; import { Stability } from 'jsii-spec'; import { Generator, GeneratorOptions } from '../generator'; +import { warn } from '../logging'; import { md2rst } from '../markdown'; import { propertySpec } from '../reflect-hacks'; import { Target, TargetOptions } from '../target'; @@ -28,6 +29,22 @@ export default class Python extends Target { // Actually package up our code, both as a sdist and a wheel for publishing. await shell("python3", ["setup.py", "sdist", "--dist-dir", outDir], { cwd: sourceDir }); await shell("python3", ["setup.py", "bdist_wheel", "--dist-dir", outDir], { cwd: sourceDir }); + if (await twineIsPresent()) { + await shell("twine", ["check", path.join(outDir, '*')], { cwd: sourceDir }); + } else { + warn('Unable to validate distribution packages because `twine` is not present. ' + + 'Run `pip3 install twine` to enable distribution package validation.'); + } + + // Approximating existence check using `pip3 show`. If that fails, assume twine is not there. + async function twineIsPresent(): Promise { + try { + const output = await shell("pip3", ["show", "twine"], { cwd: sourceDir }); + return output.trim() !== ''; + } catch { + return false; + } + } } } diff --git a/superchain/Dockerfile b/superchain/Dockerfile index 3201a2f4e7..6fbc0308e6 100644 --- a/superchain/Dockerfile +++ b/superchain/Dockerfile @@ -1,32 +1,6 @@ FROM amazonlinux:2 -# Install shared dependencies -RUN yum -y upgrade \ - && yum -y install awscli git gzip rsync tar unzip zip \ - && yum clean all && rm -rf /var/cache/yum - -### -# We'll be trying to install stuff slowest to fastest, as a courtesy to people who'll have to build & re-build this. -### - -# Install NVM and Node 8+ -ARG NODE_VERSION=8.16.0 -ARG NPM_VERSION=6.8.0 -ENV NVM_DIR=/usr/local/nvm -RUN curl -sSL https://raw.githubusercontent.com/nvm-sh/nvm/v0.34.0/install.sh -o /tmp/install-nvm.sh \ - && echo "ef7ad1db40c92f348c0461f24983b71ba0ea7d45d4007a36e484270fa7f81fcf /tmp/install-nvm.sh" | sha256sum -c \ - && mkdir -p ${NVM_DIR} \ - && bash /tmp/install-nvm.sh \ - && rm /tmp/install-nvm.sh \ - && . ${NVM_DIR}/nvm.sh \ - && nvm install ${NODE_VERSION} \ - && nvm alias default ${NODE_VERSION} \ - && nvm use default \ - && npm -g install npm@^${NPM_VERSION} \ - && npm set unsafe-perm true -ENV NODE_PATH=${NVM_DIR}/versions/node/v${NODE_VERSION}/lib/node_modules \ - PATH=${PATH}:${NVM_DIR}/versions/node/v${NODE_VERSION}/bin -# Install .NET Core & mono +# Install .NET Core, mono & PowerShell ENV DOTNET_CLI_TELEMETRY_OPTOUT=1 \ DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 COPY gpg/mono.asc /tmp/mono.asc @@ -34,16 +8,12 @@ RUN rpm --import "https://packages.microsoft.com/keys/microsoft.asc" && rpm -Uvh "https://packages.microsoft.com/config/rhel/7/packages-microsoft-prod.rpm" \ && rpm --import /tmp/mono.asc && rm -f /tmp/mono.asc \ && curl "https://download.mono-project.com/repo/centos6-stable.repo" | tee /etc/yum.repos.d/mono-centos6-stable.repo \ - && yum -y install dotnet-sdk-2.2 mono-devel \ - && yum clean all && rm -rf /var/cache/yum - -# Install Powershell -RUN yum install -y powershell \ + && yum -y install dotnet-sdk-2.2 mono-devel powershell \ && yum clean all && rm -rf /var/cache/yum # Install Python 3 -RUN yum -y install python3 python3-pip python3-wheel \ - && python3 -m pip install --upgrade pip wheel setuptools \ +RUN yum -y install python3 python3-pip \ + && python3 -m pip install --upgrade pip setuptools wheel twine \ && yum clean all && rm -rf /var/cache/yum # Install Ruby 2.4+ @@ -64,6 +34,32 @@ RUN amazon-linux-extras install docker && yum clean all && rm -rf /var/cache/yum VOLUME /var/lib/docker +# Install shared dependencies +RUN yum -y install awscli git gzip rsync tar unzip zip \ + && yum clean all && rm -rf /var/cache/yum + +# Install NVM and Node 8+ +ARG NODE_VERSION=8.16.0 +ARG NPM_VERSION=6.8.0 +ENV NVM_DIR=/usr/local/nvm +RUN curl -sSL https://raw.githubusercontent.com/nvm-sh/nvm/v0.34.0/install.sh -o /tmp/install-nvm.sh \ + && echo "ef7ad1db40c92f348c0461f24983b71ba0ea7d45d4007a36e484270fa7f81fcf /tmp/install-nvm.sh" | sha256sum -c \ + && mkdir -p ${NVM_DIR} \ + && bash /tmp/install-nvm.sh \ + && rm /tmp/install-nvm.sh \ + && . ${NVM_DIR}/nvm.sh \ + && nvm install ${NODE_VERSION} \ + && nvm alias default ${NODE_VERSION} \ + && nvm use default \ + && npm -g install npm@^${NPM_VERSION} \ + && npm set unsafe-perm true +ENV NODE_PATH=${NVM_DIR}/versions/node/v${NODE_VERSION}/lib/node_modules \ + PATH=${PATH}:${NVM_DIR}/versions/node/v${NODE_VERSION}/bin + +# Upgrade all packages that weren't up-to-date just yet +RUN yum -y upgrade \ + && yum clean all && rm -rf /var/cache/yum + # Install some configuration COPY ssh_config /root/.ssh/config COPY dockerd-entrypoint.sh /usr/local/bin/