Skip to content

Commit

Permalink
feat(python): check distribution artifacts with twine (#711)
Browse files Browse the repository at this point in the history
Adds a `twine check` invocation at the end of building the distribution
artifacts for PyPI, in order to ensure the produced bundles will render
propertly on PyPI, and be valid artifacts.

This particularly guards against "tool old" versions of `setuptools`,
but could also catch certain other issues.

Fixes #710
  • Loading branch information
RomainMuller authored and mergify[bot] committed Aug 14, 2019
1 parent b37790d commit f3d1da0
Show file tree
Hide file tree
Showing 4 changed files with 76 additions and 52 deletions.
29 changes: 17 additions & 12 deletions CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -11,20 +11,25 @@ Due to the polyglot nature of `jsii`, the toolchain requirements are somewhat
more complicated than for most projects. In order to locally develop `jsii`, you
will need the following tools:

- [Node 8.11.0] or later
- An OpenJDK-8 distribution
+ [Oracle's OpenJDK8]
+ [Amazon Corretto 8]
- [.NET Core 2.0] or later
- [Python 3.6.5] or later
- [Ruby 2.5.1] or later

[Node 8.11.0]: https://nodejs.org/download/release/v8.11.0/
- [Node `8.11.0`] or later
- An OpenJDK-8 distribution (e.g: [Oracle's OpenJDK8], [Amazon Corretto 8])
+ [`maven >= 3.0.5`](https://maven.apache.org)
- [.NET Core `2.0`] or later
+ *Recommended:* [`mono >= 5`](https://www.mono-project.com)
- [Python `3.6.5`] or later
+ [`pip`](https://pip.pypa.io/en/stable/installing/)
+ [`setuptools >= 38.6.0`](https://pypi.org/project/setuptools/)
+ [`wheel`](https://pypi.org/project/wheel/)
+ *Recommended:* [`twine`](https://pypi.org/project/twine/)
- [Ruby `2.4.4p296`] or later
+ [`bundler ~> 1.17.2`](https://bundler.io)

[Node `8.11.0`]: https://nodejs.org/download/release/v8.11.0/
[Oracle's OpenJDK8]: http://openjdk.java.net/install/
[Amazon Corretto 8]: https://aws.amazon.com/corretto/
[.NET Core 2.0]: https://www.microsoft.com/net/download
[Python 3.6.5]: https://www.python.org/downloads/release/python-365/
[Ruby 2.5.1]: https://www.ruby-lang.org/en/news/2018/03/28/ruby-2-5-1-released/
[.NET Core `2.0`]: https://www.microsoft.com/net/download
[Python `3.6.5`]: https://www.python.org/downloads/release/python-365/
[Ruby `2.4.4p296`]: https://www.ruby-lang.org/en/news/2018/03/28/ruby-2-5-1-released/

### Alterative: build in Docker

Expand Down
18 changes: 12 additions & 6 deletions packages/jsii-pacmak/lib/logging.ts
Original file line number Diff line number Diff line change
@@ -1,14 +1,20 @@
export let level = 0;

export const LEVEL_INFO = 1;
export const LEVEL_VERBOSE = 2;

export enum Level {
WARN = -1,
QUIET = 0,
INFO = 1,
VERBOSE = 2,
}

export const LEVEL_INFO: number = Level.INFO;
export const LEVEL_VERBOSE: number = Level.VERBOSE;

/** The minimal logging level for messages to be emitted. */
export let level = Level.QUIET;

export function warn(fmt: string, ...args: any[]) {
log(Level.WARN, fmt, ...args);
}

export function info(fmt: string, ...args: any[]) {
log(Level.INFO, fmt, ...args);
}
Expand All @@ -23,4 +29,4 @@ function log(messageLevel: Level, fmt: string, ...args: any[]) {
// tslint:disable-next-line:no-console
console.error.call(console, ...[ `[jsii-pacmak] [${levelName}]`, fmt, ...args ]);
}
}
}
17 changes: 17 additions & 0 deletions packages/jsii-pacmak/lib/targets/python.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import * as reflect from 'jsii-reflect';
import * as spec from 'jsii-spec';
import { Stability } from 'jsii-spec';
import { Generator, GeneratorOptions } from '../generator';
import { warn } from '../logging';
import { md2rst } from '../markdown';
import { propertySpec } from '../reflect-hacks';
import { Target, TargetOptions } from '../target';
Expand All @@ -28,6 +29,22 @@ export default class Python extends Target {
// Actually package up our code, both as a sdist and a wheel for publishing.
await shell("python3", ["setup.py", "sdist", "--dist-dir", outDir], { cwd: sourceDir });
await shell("python3", ["setup.py", "bdist_wheel", "--dist-dir", outDir], { cwd: sourceDir });
if (await twineIsPresent()) {
await shell("twine", ["check", path.join(outDir, '*')], { cwd: sourceDir });
} else {
warn('Unable to validate distribution packages because `twine` is not present. '
+ 'Run `pip3 install twine` to enable distribution package validation.');
}

// Approximating existence check using `pip3 show`. If that fails, assume twine is not there.
async function twineIsPresent(): Promise<boolean> {
try {
const output = await shell("pip3", ["show", "twine"], { cwd: sourceDir });
return output.trim() !== '';
} catch {
return false;
}
}
}
}

Expand Down
64 changes: 30 additions & 34 deletions superchain/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,49 +1,19 @@
FROM amazonlinux:2

# Install shared dependencies
RUN yum -y upgrade \
&& yum -y install awscli git gzip rsync tar unzip zip \
&& yum clean all && rm -rf /var/cache/yum

###
# We'll be trying to install stuff slowest to fastest, as a courtesy to people who'll have to build & re-build this.
###

# Install NVM and Node 8+
ARG NODE_VERSION=8.16.0
ARG NPM_VERSION=6.8.0
ENV NVM_DIR=/usr/local/nvm
RUN curl -sSL https://raw.githubusercontent.com/nvm-sh/nvm/v0.34.0/install.sh -o /tmp/install-nvm.sh \
&& echo "ef7ad1db40c92f348c0461f24983b71ba0ea7d45d4007a36e484270fa7f81fcf /tmp/install-nvm.sh" | sha256sum -c \
&& mkdir -p ${NVM_DIR} \
&& bash /tmp/install-nvm.sh \
&& rm /tmp/install-nvm.sh \
&& . ${NVM_DIR}/nvm.sh \
&& nvm install ${NODE_VERSION} \
&& nvm alias default ${NODE_VERSION} \
&& nvm use default \
&& npm -g install npm@^${NPM_VERSION} \
&& npm set unsafe-perm true
ENV NODE_PATH=${NVM_DIR}/versions/node/v${NODE_VERSION}/lib/node_modules \
PATH=${PATH}:${NVM_DIR}/versions/node/v${NODE_VERSION}/bin
# Install .NET Core & mono
# Install .NET Core, mono & PowerShell
ENV DOTNET_CLI_TELEMETRY_OPTOUT=1 \
DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1
COPY gpg/mono.asc /tmp/mono.asc
RUN rpm --import "https://packages.microsoft.com/keys/microsoft.asc" \
&& rpm -Uvh "https://packages.microsoft.com/config/rhel/7/packages-microsoft-prod.rpm" \
&& rpm --import /tmp/mono.asc && rm -f /tmp/mono.asc \
&& curl "https://download.mono-project.com/repo/centos6-stable.repo" | tee /etc/yum.repos.d/mono-centos6-stable.repo \
&& yum -y install dotnet-sdk-2.2 mono-devel \
&& yum clean all && rm -rf /var/cache/yum

# Install Powershell
RUN yum install -y powershell \
&& yum -y install dotnet-sdk-2.2 mono-devel powershell \
&& yum clean all && rm -rf /var/cache/yum

# Install Python 3
RUN yum -y install python3 python3-pip python3-wheel \
&& python3 -m pip install --upgrade pip wheel setuptools \
RUN yum -y install python3 python3-pip \
&& python3 -m pip install --upgrade pip setuptools wheel twine \
&& yum clean all && rm -rf /var/cache/yum

# Install Ruby 2.4+
Expand All @@ -64,6 +34,32 @@ RUN amazon-linux-extras install docker
&& yum clean all && rm -rf /var/cache/yum
VOLUME /var/lib/docker

# Install shared dependencies
RUN yum -y install awscli git gzip rsync tar unzip zip \
&& yum clean all && rm -rf /var/cache/yum

# Install NVM and Node 8+
ARG NODE_VERSION=8.16.0
ARG NPM_VERSION=6.8.0
ENV NVM_DIR=/usr/local/nvm
RUN curl -sSL https://raw.githubusercontent.com/nvm-sh/nvm/v0.34.0/install.sh -o /tmp/install-nvm.sh \
&& echo "ef7ad1db40c92f348c0461f24983b71ba0ea7d45d4007a36e484270fa7f81fcf /tmp/install-nvm.sh" | sha256sum -c \
&& mkdir -p ${NVM_DIR} \
&& bash /tmp/install-nvm.sh \
&& rm /tmp/install-nvm.sh \
&& . ${NVM_DIR}/nvm.sh \
&& nvm install ${NODE_VERSION} \
&& nvm alias default ${NODE_VERSION} \
&& nvm use default \
&& npm -g install npm@^${NPM_VERSION} \
&& npm set unsafe-perm true
ENV NODE_PATH=${NVM_DIR}/versions/node/v${NODE_VERSION}/lib/node_modules \
PATH=${PATH}:${NVM_DIR}/versions/node/v${NODE_VERSION}/bin

# Upgrade all packages that weren't up-to-date just yet
RUN yum -y upgrade \
&& yum clean all && rm -rf /var/cache/yum

# Install some configuration
COPY ssh_config /root/.ssh/config
COPY dockerd-entrypoint.sh /usr/local/bin/
Expand Down

0 comments on commit f3d1da0

Please sign in to comment.