diff --git a/.github/workflows/publish-docker.yml b/.github/workflows/publish-docker.yml new file mode 100644 index 000000000..d05cc8c78 --- /dev/null +++ b/.github/workflows/publish-docker.yml @@ -0,0 +1,52 @@ +name: 'Publish Docker' + +on: + workflow_dispatch: + inputs: + tag: + description: 'Tag to publish Docker image for (scip-ruby-vM.N.P)' + workflow_run: + workflows: ['Release'] + types: + - completed + +jobs: + release-image: + runs-on: ubuntu-latest + steps: + - run: | + if [ -z "$TAG" ]; then + # From https://stackoverflow.com/questions/10649814/get-last-git-tag-from-a-remote-repo-without-cloning + TAG="$(git -c 'versionsort.suffix=-' \ + ls-remote --exit-code --refs --sort='version:refname' --tags https://github.com/sourcegraph/scip-ruby.git 'scip-ruby-v*' \ + | tail --lines=1 \ + | cut --delimiter='/' --fields=3)" + fi + { + echo "TAG=$TAG" + echo "PATCH=${TAG/scip-ruby-v/}" + echo "MINOR=${PATCH%.*}" + echo "MAJOR=${MINOR%.*}" + } >> "$GITHUB_ENV" + env: + TAG: ${{ inputs.tag }} + - uses: actions/checkout@v3 + with: + ref: ${{ env.TAG }} + - uses: docker/setup-buildx-action@v2 + - name: Login to DockerHub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - name: Build and push + id: docker_build_autoindex + uses: docker/build-push-action@v3 + with: + file: Dockerfile.autoindex + push: true + tags: | + sourcegraph/scip-ruby:autoindex + sourcegraph/scip-typescript:autoindex-${{ env.PATCH }} + sourcegraph/scip-typescript:autoindex-${{ env.MINOR }} + sourcegraph/scip-typescript:autoindex-${{ env.MAJOR }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2a70997b9..d037460ef 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,4 +1,4 @@ -name: Release +name: Release # This name is used by the publish-docker workflow on: push: tags: diff --git a/Dockerfile.autoindex b/Dockerfile.autoindex new file mode 100644 index 000000000..e172a163e --- /dev/null +++ b/Dockerfile.autoindex @@ -0,0 +1,20 @@ +FROM --platform=linux/amd64 ruby:2.7.6-alpine3.16@sha256:b014cf3e792d7130daec772241a211c40be009fc7f00e2b728ffe26805649575 + +# This Docker image is meant for auto-indexing support in Sourcegraph +# and is not recommended for third-party use. + +# gcompat is a glibc-musl compat library (scip-ruby links in glibc) +# Other deps are to help build C extensions in gems. +RUN apk add --no-cache bash wget make libstdc++ gcc g++ automake autoconf gcompat + +# Use a release binary instead of building from source +# because release builds are very time-consuming. +# +# The release version is verified by tools/scripts/publish-scip-ruby.sh +RUN wget https://github.com/sourcegraph/scip-ruby/releases/download/scip-ruby-v0.3.0/scip-ruby-x86_64-linux -O /usr/bin/scip-ruby && chmod +x /usr/bin/scip-ruby + +COPY scip_indexer/autoindex.sh /usr/bin/scip-ruby-autoindex + +RUN chmod +x /usr/bin/scip-ruby-autoindex + +CMD ["/bin/sh"] diff --git a/docs/scip-ruby/CONTRIBUTING.md b/docs/scip-ruby/CONTRIBUTING.md index 78d7cc749..11b9296a7 100644 --- a/docs/scip-ruby/CONTRIBUTING.md +++ b/docs/scip-ruby/CONTRIBUTING.md @@ -27,6 +27,7 @@ see the [Design Decisions doc][]. - [Creating PRs](#creating-prs) - [Syncing Sorbet upstream](#syncing-sorbet-upstream) - [Cutting a release](#cutting-a-release) +- [Auto-indexing support](#auto-indexing-support) - Troubleshooting - [Known build issues][] - [Known RubyGems related issues](#known-rubygems-related-issues) @@ -363,6 +364,7 @@ This will correctly use the `scip-ruby/master` branch as the target. 1. Add release notes to the [CHANGELOG](/CHANGELOG.md). 2. Bump `scip_ruby_version` in `SCIPIndexer.cc`. +3. Bump the release version in `Dockerfile.autoindex`. Run the release script: @@ -374,6 +376,36 @@ If there are any errors, fix those and re-run. A CI job will be kicked off to trigger a release. See the [release workflow](/.github/workflows/release.yml) for details. +## Auto-indexing support + +scip-ruby includes has a [Docker image](/Dockerfile.autoindex) for auto-indexing support in Sourcegraph. +From my testing, this doesn't work under Docker on macOS for some reason +with a loading error (despite accounting for the glibc-musl mismatch with gcompat). +It does work on Linux. You can build the Docker image for testing using: + +```bash +docker build . --file Dockerfile.autoindex --platform linux/amd64 --tag scip-ruby-testing +docker run --platform linux/amd64 --rm -it scip-ruby-testing /bin/bash +``` + +Auto-indexing is controlled via a [wrapper script](/scip_indexer/autoindex.sh), +so that we can handle special cases like RubyGems packages in Sourcegraph, +without putting that logic into the indexer itself. + +You can mimic the workflow of auto-indexing a package by running: + +```bash +PACKAGE=activesupport-7.0.1 +mkdir "$PACKAGE" && cd "$PACKAGE" +wget "https://rubygems.org/gems/$PACKAGE.gem" +tar -xf "$PACKAGE.gem" +tar -xzf data.tar.gz && gunzip metadata.gz && mv metadata rubygems-metadata.yml + +scip-ruby-autoindex +``` + +For a source repo, cloning the repo and running `scip-ruby-autoindex` should do the trick. + ## Troubleshooting ### Known build issues diff --git a/scip_indexer/autoindex.sh b/scip_indexer/autoindex.sh new file mode 100755 index 000000000..9fbd9a8d1 --- /dev/null +++ b/scip_indexer/autoindex.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash + +set -euo pipefail +set -x + +while [[ $# -gt 0 ]]; do + case $1 in + --gem-name) + GEM_NAME="$2" + shift # past argument + shift # past value + ;; + --gem-version) + GEM_VERSION="$2" + shift # past argument + shift # past value + ;; + --is-package) + IS_PACKAGE_REPO="YES" + shift # past argument + esac +done + +initialize_sorbet() { + gem install "$GEM_NAME" -v "$GEM_VERSION" + + # Assume that current directory is the gem root. + # `gem lock` will emit the current gem too, but we don't want that, + # as the root already has the source code. + # + # FIXME: Allow passing in gem source from the outside + # Blocked on https://github.com/sourcegraph/sourcegraph/issues/44204 + gem lock "$GEM_NAME-$GEM_VERSION" \ + | grep -v "gem '$GEM_NAME'" \ + | sed -E 's|^gem|source "https://rubygems.org"\ngem|' \ + > Gemfile + + echo "group :development do + gem 'tapioca', require: false +end" >> Gemfile + + bundle install + bundle exec tapioca init +} + +# FIXME: Modify the global gem sources list to use the configured host +# in Sourcegraph rather than https://rubygems.org + +# Created by Sourcegraph to indicate package repos. +if [ -f "rubygems-metadata.yml" ]; then + IS_PACKAGE_REPO="YES" +fi + +if [ -n "${IS_PACKAGE_REPO:-}" ]; then + if [ -z "${GEM_NAME:-}" ]; then + GEM_NAME="$(grep 'name:' rubygems-metadata.yml | head -n 1 | sed -e 's/name: //')" + fi + if [ -z "${GEM_VERSION:-}" ]; then + GEM_VERSION="$(grep ' version:' rubygems-metadata.yml | head -n 1 | sed -e 's/ version: //')" + fi + initialize_sorbet + set +e + scip-ruby . --gem-metadata "$GEM_NAME@$GEM_VERSION" + exit 0 +fi + +if [ -f "sorbet/config" ]; then + set +e + if [ -n "${GEM_NAME:-}" ] && [ -n "${GEM_VERSION:-}" ]; then + scip-ruby --gem-metadata "$GEM_NAME@$GEM_VERSION" + else + scip-ruby + fi + exit 0 +fi + + +set +e +# Can remove this once the 'default to using .' problem in +# https://github.com/sourcegraph/scip-ruby/issues/133 is fixed. +if [ -n "${GEM_NAME:-}" ] && [ -n "${GEM_VERSION:-}" ]; then + scip-ruby . --gem-metadata "$GEM_NAME@$GEM_VERSION" +else + scip-ruby . +fi diff --git a/tools/scripts/publish-scip-ruby.sh b/tools/scripts/publish-scip-ruby.sh index ccf91757a..99cabde56 100755 --- a/tools/scripts/publish-scip-ruby.sh +++ b/tools/scripts/publish-scip-ruby.sh @@ -21,6 +21,11 @@ if ! grep -q "const char scip_ruby_version\[\] = \"$NEW_VERSION\"" scip_indexer/ exit 1 fi +if ! grep -q "download/scip-ruby-$NEW_VERSION" Dockerfile.autoindex; then + echo "error: scip-ruby version in Dockerfile is not the latest release version." + exit 1 +fi + if ! git diff --quiet; then echo "error: Found unstaged changes; aborting." exit 1