Skip to content

Commit

Permalink
Enable running E2E tests from streamlit/streamlit (#72)
Browse files Browse the repository at this point in the history
  • Loading branch information
sfc-gh-gpijanski committed Aug 31, 2023
1 parent c1b48ea commit 6a95382
Show file tree
Hide file tree
Showing 7 changed files with 276 additions and 110 deletions.
57 changes: 57 additions & 0 deletions .github/actions/build_component_wheels/action.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
name: 'Build component wheels'
description: 'Action for building all components wheels'
inputs:
custom_streamlit_component_lib_file:
required: false
description: |
Path to local Streamlit Component Library package.
To generate a package, run the npm pack command in the directory with the Streamlit component library.
default: ''

outputs:
output_directory:
description: 'The directory where the built wheel packages of all the components are located'
value: ${{ steps.final_step.outputs.output_directory }}

runs:
using: 'composite'
steps:
- name: Link develop version of streamlit-component-lib
if: inputs.custom_streamlit_component_lib_file != ''
working-directory: ${{ github.action_path }}/../../../
shell: bash
env:
STREAMLIT_COMPONENT_LIB_FILE: ${{ inputs.custom_streamlit_component_lib_file }}
run: |
find examples template template-reactless -name frontend -maxdepth 3 | while IFS= read -r line; do
(
cd "${line}";
npm install "${STREAMLIT_COMPONENT_LIB_FILE}"
)
done
- name: Install node dependencies
working-directory: ${{ github.action_path }}/../../../
shell: bash
run: ./dev.py all-npm-install

- name: Build frontend code
working-directory: ${{ github.action_path }}/../../../
shell: bash
run: ./dev.py all-npm-build

- name: Build wheel packages
working-directory: ${{ github.action_path }}/../../../
shell: bash
run: |
find examples template template-reactless -maxdepth 3 -name '__init__.py' |\
xargs -n 1 sed -i 's/_RELEASE = False/_RELEASE = True/';
./dev.py all-python-build-package
- name: Set outputs
working-directory: ${{ github.action_path }}/../../../
shell: bash
id: final_step
run: |
output_dir="$(readlink -e dist)"
echo "output_directory=${output_dir}" >> $GITHUB_OUTPUT
54 changes: 54 additions & 0 deletions .github/actions/pack_streamlit_component_library/action.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
name: 'Pack Streamlit Component Library'
description: 'Packing Streamlit Component Library'
inputs:
working_directory:
required: false
description: 'Directory where the streamlit/streamlit repository will be checked out'
default: 'streamlit'

outputs:
output_file:
description: 'Packed Streamlit Component Library file'
value: ${{ steps.final_step.outputs.output_file }}

runs:
using: 'composite'
steps:
- name: Check prerequisite
shell: bash
run: |
if ! command -v node > /dev/null
then
echo "Node is required to use this action"
exit 1
fi
- name: Checkout streamlit/streamlit
uses: actions/checkout@v3
with:
persist-credentials: false
repository: streamlit/streamlit
ref: develop
path: ${{ inputs.working_directory }}

- name: Install node dependencies for streamlit-component-lib
working-directory: ${{ inputs.working_directory }}/component-lib/
shell: bash
run: |
npm install -g yarn
yarn install
- name: Pack streamlit-component-lib package
working-directory: ${{ inputs.working_directory }}/component-lib/
shell: bash
run: yarn run build && npm pack

- name: Link develop version of streamlit-component-lib
if: inputs.custom_streamlit_component_lib_file != ''
working-directory: ${{ inputs.working_directory }}/component-lib/
id: final_step
shell: bash
run: |
component_lib_tar_gz=$(find "./" -maxdepth 1 -name 'streamlit-component-lib-*.tgz')
component_lib_tar_gz=$(readlink -e "${component_lib_tar_gz}")
echo "output_file=${component_lib_tar_gz}" >> $GITHUB_OUTPUT
46 changes: 46 additions & 0 deletions .github/actions/run_e2e/action.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
name: E2E tests
description: Run e2e tests for templates and examples in separate docker images

inputs:
python_version:
description: 'A Python version used to create a Docker Image'
required: true
default: '3.8'
streamlit_version:
description: 'Streamlit version to be installed in a Docker Image'
required: false
default: ''
streamlit_wheel_file:
description: 'Wheel file of a development version Streamlit to be installed in a Docker Image'
required: false
default: ''


runs:
using: 'composite'
steps:
- name: Build docker images
run: |
if [ -n "${{ env.STREAMLIT_VERSION }}" ]; then
./dev.py e2e-build-images "--streamlit-version=${{ env.STREAMLIT_VERSION }}" "--python-version=${{ env.PYTHON_VERSION }}"
elif [ -n "${{ env.STREAMLIT_WHEEL_FILE }}" ]; then
./dev.py e2e-build-images "--streamlit-wheel-file=${{ env.STREAMLIT_WHEEL_FILE }}" "--python-version=${{ env.PYTHON_VERSION }}"
else
echo "One and exactly one input is required: streamlit_version, streamlit_wheel_file".
exit 1
fi
shell: bash
working-directory: ${{ github.action_path }}/../../../
env:
STREAMLIT_VERSION: ${{ inputs.streamlit_version }}
STREAMLIT_WHEEL_FILE: ${{ inputs.streamlit_wheel_file }}
PYTHON_VERSION: ${{ inputs.python_version }}


- name: Run e2e tests
run: ./dev.py e2e-run-tests "--streamlit-version=${{ env.STREAMLIT_VERSION }}" "--python-version=${{ env.PYTHON_VERSION }}"
shell: bash
working-directory: ${{ github.action_path }}/../../../
env:
STREAMLIT_VERSION: ${{ inputs.streamlit_wheel_file == '' && inputs.streamlit_version || 'custom' }}
PYTHON_VERSION: ${{ inputs.python_version }}
75 changes: 18 additions & 57 deletions .github/workflows/ci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ jobs:
- uses: actions/checkout@v3
with:
persist-credentials: false
path: component-template

- name: Use Node.js ${{ env.NODE_VERSION }}
uses: actions/setup-node@v3
Expand All @@ -50,77 +51,37 @@ jobs:

- name: Check dependencies for examples
run: ./dev.py examples-check-deps
working-directory: ./component-template/

- name: Check e2e utils files
run: ./dev.py e2e-utils-check
working-directory: ./component-template/

- name: Checkout streamlit/streamlit
- name: Pack Streamlit Component Library
uses: ./component-template/.github/actions/pack_streamlit_component_library
if: matrix.component_lib_version == 'develop'
uses: actions/checkout@v3
with:
persist-credentials: false
repository: streamlit/streamlit
ref: develop
path: streamlit

- name: Install node dependencies for streamlit-component-lib
if: matrix.component_lib_version == 'develop'
working-directory: ./streamlit/component-lib/
run: |
npm install -g yarn
yarn install
- name: Build streamlit-component-lib package
if: matrix.component_lib_version == 'develop'
working-directory: ./streamlit/component-lib/
run: yarn run build && npm pack

- name: Link develop version of streamlit-component-lib
if: matrix.component_lib_version == 'develop'
env:
COMPONENT_LIB_DIR: ${{ github.workspace }}/streamlit/component-lib/
run: |
component_lib_tar_gz=$(find "${COMPONENT_LIB_DIR}" -maxdepth 1 -name 'streamlit-component-lib-*.tgz')
component_lib_tar_gz=$(readlink -e "${component_lib_tar_gz}")
find examples template template-reactless -name frontend -maxdepth 3 | while IFS= read -r line; do
(
cd "${line}";
npm install "${component_lib_tar_gz}"
)
done
id: streamlit_component_library

- name: Install node dependencies
run: ./dev.py all-npm-install

- name: Build frontend code
run: ./dev.py all-npm-build

- name: Build wheel packages
run: |
find examples template template-reactless -maxdepth 3 -name '__init__.py' |\
xargs -n 1 sed -i 's/_RELEASE = False/_RELEASE = True/';
./dev.py all-python-build-package
- name: Build components wheels
uses: ./component-template/.github/actions/build_component_wheels
id: component_wheels
with:
custom_streamlit_component_lib_file: >-
${{ matrix.component_lib_version == 'develop' && steps.streamlit_component_library.outputs.output_file || '' }}
- name: Upload wheel packages
uses: actions/upload-artifact@v3
with:
name: all-wheel
path: dist/*.whl
path: ${{ steps.component_wheels.outputs.output_directory }}/*.whl
if-no-files-found: error

- name: Set up Docker Buildx
- name: Run E2E tests
if: matrix.node_version == '19.x'
uses: docker/setup-buildx-action@7703e82fbced3d0c9eec08dff4429c023a5fd9a9 # v2.9.1

- name: Build docker images
if: matrix.node_version == '19.x'
run: ./dev.py e2e-build-images "--streamlit-version=${{ env.STREAMLIT_VERSION }}" "--python-version=${{ env.PYTHON_VERSION }}"

- name: Run e2e tests
if: matrix.node_version == '19.x'
run: ./dev.py e2e-run-tests "--streamlit-version=${{ env.STREAMLIT_VERSION }}" "--python-version=${{ env.PYTHON_VERSION }}"
uses: ./component-template/.github/actions/run_e2e
with:
python_version: ${{ env.PYTHON_VERSION }}
streamlit_version: ${{ env.STREAMLIT_VERSION }}

build-cookiecutter:
runs-on: ubuntu-latest
Expand Down
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -68,3 +68,4 @@ template-reactless/**/package-lock.json
# VSCode
########################################################################
.vscode/
buildcontext/
29 changes: 25 additions & 4 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
# syntax=docker/dockerfile:1.4

ARG PYTHON_VERSION="3.11.4"
FROM python:${PYTHON_VERSION}-slim-bullseye
ARG PYTHON_VERSION
FROM python:${PYTHON_VERSION}-slim-bullseye as e2e_base

SHELL ["/bin/bash", "-o", "pipefail", "-e", "-u", "-x", "-c"]

Expand All @@ -11,11 +11,19 @@ ENV PIP_VERSION=${PIP_VERSION}

RUN pip install --no-cache-dir --upgrade "pip==${PIP_VERSION}" && pip --version

# Setup Playwright
ARG PLAYWRIGHT_VERSION="1.36.0"
ENV PLAYWRIGHT_VERSION=${PLAYWRIGHT_VERSION}

RUN pip install --no-cache-dir playwright=="${PLAYWRIGHT_VERSION}" && playwright install webkit chromium firefox --with-deps

ENV PYTHONUNBUFFERED=1
ENV PIP_ROOT_USER_ACTION=ignore
RUN mkdir /component
WORKDIR /component

FROM e2e_base AS e2e_pip

# Install streamlit and components
ARG STREAMLIT_VERSION="latest"
ENV E2E_STREAMLIT_VERSION=${STREAMLIT_VERSION}
Expand All @@ -24,7 +32,7 @@ RUN <<"EOF"
if [[ "${E2E_STREAMLIT_VERSION}" == "latest" ]]; then
pip install --no-cache-dir "streamlit"
elif [[ "${E2E_STREAMLIT_VERSION}" == "nightly" ]]; then
pip uninstall --yes streamlit
pip uninstall --yes "streamlit"
pip install --no-cache-dir "streamlit-nightly"
else
pip install --no-cache-dir "streamlit==${E2E_STREAMLIT_VERSION}"
Expand All @@ -38,4 +46,17 @@ if [[ "${E2E_STREAMLIT_VERSION}" == "nightly" ]]; then
else
echo "${installed_streamlit_version}" | grep -v 'dev'
fi
EOF
EOF

FROM e2e_base AS e2e_whl

ARG STREAMLIT_WHEEL_PATH

RUN --mount=type=bind,source=./buildcontext,target=/buildcontext <<"EOF"
pip uninstall --yes "streamlit" "streamlit-nightly"
find /buildcontext/ -name '*.whl'| xargs -t pip install --no-cache-dir

# Coherence check
installed_streamlit_version=$(python -c "import streamlit; print(streamlit.__version__)")
echo "Installed Streamlit version: ${installed_streamlit_version}"
EOF

0 comments on commit 6a95382

Please sign in to comment.