[LAB-635] private ipfs compose (#677)

Co-authored-by: thetechnocrat-dev <josh.mcmenemy@openzyme.bio>
labdao · Oct 4, 2023 · d735d1d · d735d1d
1 parent 7dc0233
commit d735d1d
Show file tree

Hide file tree

Showing 3 changed files with 183 additions and 63 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -1,3 +1,5 @@
+---
+
 # This workflow will build a golang project and runs CI tests
 
 name: CI
@@ -21,7 +23,10 @@ concurrency:
 
 jobs:
   go:
-    runs-on: ubuntu-22.04
+    strategy:
+      matrix:
+        os: [ubuntu-22.04] # , macos-13]
+    runs-on: ${{ matrix.os }}
     environment: ci
     steps:
       - uses: actions/checkout@v3
@@ -40,64 +45,133 @@ jobs:
       - name: upload plex binary to be used by other jobs
         uses: actions/upload-artifact@v3
         with:
-          name: plex-binary
+          name: plex-binary-${{matrix.os}}
           path: ./plex
 
       - name: Test
         run: go test ./... -v
 
+  # Only downloading cli for linux
   ci-setup-compose:
-    runs-on: ubuntu-22.04
+    strategy:
+      matrix:
+        os: [ubuntu-22.04]
+    runs-on: ${{ matrix.os }}
     environment: ci
     steps:
       - name: Download docker compose
         run: |
           # Download docker-compose plugin
-          curl -sSL https://github.com/docker/compose/releases/download/v2.20.3/docker-compose-linux-x86_64 -o docker-compose
+          if [[ "$OSTYPE" == "linux-gnu"* ]]; then
+            curl -sSL https://github.com/docker/compose/releases/download/v2.20.3/docker-compose-linux-x86_64 -o docker-compose
+          elif [[ "$OSTYPE" == "darwin"* ]]; then
+            curl -sSL https://github.com/docker/compose/releases/download/v2.20.3/docker-compose-darwin-aarch64 -o docker-compose
+          fi
 
       - name: upload docker-compose plugin to be used later
         uses: actions/upload-artifact@v3
         with:
-          name: docker-compose-plugin
+          name: docker-compose-plugin-${{matrix.os}}
           path: ./docker-compose
 
   ci-setup-ipfs:
-    runs-on: ubuntu-22.04
+    strategy:
+      matrix:
+        os: [ubuntu-22.04] # , macos-13]
+    runs-on: ${{ matrix.os }}
     environment: ci
     steps:
       - name: Download and extract ipfs binary tarball
         run: |
           # Download ipfs binary
-          curl -sSL https://github.com/ipfs/kubo/releases/download/v0.18.0/kubo_v0.18.0_linux-amd64.tar.gz -o kubo.tgz
+          # Download docker-compose plugin
+          if [[ "$OSTYPE" == "linux-gnu"* ]]; then
+            curl -sSL https://github.com/ipfs/kubo/releases/download/v0.18.0/kubo_v0.18.0_linux-amd64.tar.gz -o kubo.tgz
+          elif [[ "$OSTYPE" == "darwin"* ]]; then
+            curl -sSL https://github.com/ipfs/kubo/releases/download/v0.18.0/kubo_v0.18.0_darwin-arm64.tar.gz -o kubo.tgz
+          fi
           tar -zxvf kubo.tgz
 
       - name: upload ipfs binary to be used later
         uses: actions/upload-artifact@v3
         with:
-          name: ipfs-binary
+          name: ipfs-binary-${{matrix.os}}
           path: ./kubo/ipfs
+
+  # Download Bacalhau CLI for troubleshooting
+  ci-setup-bacalhau:
+    strategy:
+      matrix:
+        os: [ubuntu-22.04]
+    runs-on: ${{ matrix.os }}
+    environment: ci
+    steps:
+      - name: Download bacalhau
+        run: |
+          # Download bacalhau plugin
+          if [[ "$OSTYPE" == "linux-gnu"* ]]; then
+            curl -sSL https://github.com/bacalhau-project/bacalhau/releases/download/v1.0.3/bacalhau_v1.0.3_linux_amd64.tar.gz -o bacalhau.tgz
+          elif [[ "$OSTYPE" == "darwin"* ]]; then
+            curl -sSL https://github.com/bacalhau-project/bacalhau/releases/download/v1.0.3/bacalhau_v1.0.3_darwin_arm64.tar.gz -o bacalhau.tgz
+          fi
+          tar -zxvf bacalhau.tgz
+
+      - name: upload bacalhau plugin to be used later
+        uses: actions/upload-artifact@v3
+        with:
+          name: bacalhau-binary-${{matrix.os}}
+          path: ./bacalhau
 
   ci-public:
     needs:
       - go
       - ci-setup-compose
-    runs-on: ubuntu-22.04
+      - ci-setup-bacalhau
+    strategy:
+      matrix:
+        os: [ubuntu-22.04] # , macos-13]
+    runs-on: ${{ matrix.os }}
     environment: ci
+    env:
+      # Setting it at workflow level to be used by all the steps
+      BACALHAU_API_HOST: "127.0.0.1"
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
 
       - name: download plex binary
         uses: actions/download-artifact@v3
         with:
-          name: plex-binary
+          name: plex-binary-${{matrix.os}}
+
+      - name: download bacalhau binary
+        uses: actions/download-artifact@v3
+        with:
+          name: bacalhau-binary-${{matrix.os}}
 
       - name: download docker compose plugin artifact
+        if: runner.os != 'macos'
         uses: actions/download-artifact@v3
         with:
-          name: docker-compose-plugin
+          name: docker-compose-plugin-${{matrix.os}}
+
+      - name: Setup docker (missing on MacOS)
+        if: runner.os == 'macos'
+        run: |
+          brew install colima docker docker-compose
+          colima start
+
+          # For testcontainers to find the Colima socket
+          # https://github.com/abiosoft/colima/blob/main/docs/FAQ.md#cannot-connect-to-the-docker-daemon-at-unixvarrundockersock-is-the-docker-daemon-running
+          sudo ln -sf $HOME/.colima/default/docker.sock /var/run/docker.sock
+
+          mkdir -p ~/.docker/cli-plugins
+          ln -sfn /usr/local/opt/docker-compose/bin/docker-compose ~/.docker/cli-plugins/docker-compose
+          docker version
+          docker compose version
 
       - name: Setup docker compose plugin
+        if: runner.os != 'macos'
         run: |
           # Install docker-compose
           DOCKER_CONFIG=${DOCKER_CONFIG:-$HOME/.docker}
@@ -124,9 +198,6 @@ jobs:
           # Add execute permission
           chmod +x plex
 
-          # Run tests against it
-          export BACALHAU_API_HOST=127.0.0.1
-
           result_dir=$(./plex init -t tools/equibind.json -i '{"protein": ["testdata/binding/abl/7n9g.pdb"], "small_molecule": ["testdata/binding/abl/ZINC000003986735.sdf"]}' --scatteringMethod=dotProduct --autoRun=true -a test -a ci | grep 'Finished processing, results written to' | sed -n 's/^.*Finished processing, results written to //p' | sed 's/\/io.json//')
           cd "$result_dir/entry-0/outputs"
           if [ "$(find . -name '*docked.sdf' | grep 'docked.sdf')" == "" ]; then
@@ -137,10 +208,21 @@ jobs:
             find . -name '*docked.sdf' | grep 'docked.sdf'
           fi
 
+      - name: Run bacalhau describe
+        # run always even when
+        if: always()
+        run: |
+          # Execute permission
+          chmod +x ./bacalhau
+          JOB_ID=$(./bacalhau list --output json | jq -r '.[0].State.JobID')
+          ./bacalhau describe ${JOB_ID}
+
       - name: upload outputs
+        # run always even when
+        if: always()
         uses: actions/upload-artifact@v3
         with:
-          name: ci-public-output
+          name: ci-public-output-${{matrix.os}}
           path: |
             job-*
             jobs/
@@ -150,28 +232,57 @@ jobs:
       - go
       - ci-setup-compose
       - ci-setup-ipfs
-    runs-on: ubuntu-22.04
+      - ci-setup-bacalhau
+    strategy:
+      matrix:
+        os: [ubuntu-22.04] # , macos-13]
+    runs-on: ${{ matrix.os }}
     environment: ci
+    env:
+      # Setting it at workflow level to be used by all the steps
+      BACALHAU_API_HOST: "127.0.0.1"
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
 
       - name: download plex binary
         uses: actions/download-artifact@v3
         with:
-          name: plex-binary
+          name: plex-binary-${{matrix.os}}
 
-      - name: download docker compose plugin artifact
+      - name: download ipfs binary
         uses: actions/download-artifact@v3
         with:
-          name: docker-compose-plugin
+          name: ipfs-binary-${{matrix.os}}
 
-      - name: download ipfs binary
+      - name: download bacalhau binary
         uses: actions/download-artifact@v3
         with:
-          name: ipfs-binary
+          name: bacalhau-binary-${{matrix.os}}
+
+      - name: download docker compose plugin artifact
+        if: runner.os != 'macos'
+        uses: actions/download-artifact@v3
+        with:
+          name: docker-compose-plugin-${{matrix.os}}
+
+      - name: Setup docker (missing on MacOS)
+        if: runner.os == 'macos'
+        run: |
+          brew install colima docker docker-compose
+          colima start
+
+          # For testcontainers to find the Colima socket
+          # https://github.com/abiosoft/colima/blob/main/docs/FAQ.md#cannot-connect-to-the-docker-daemon-at-unixvarrundockersock-is-the-docker-daemon-running
+          sudo ln -sf $HOME/.colima/default/docker.sock /var/run/docker.sock
+
+          mkdir -p ~/.docker/cli-plugins
+          ln -sfn /usr/local/opt/docker-compose/bin/docker-compose ~/.docker/cli-plugins/docker-compose
+          docker version
+          docker compose version
 
       - name: Setup docker compose plugin
+        if: runner.os != 'macos'
         run: |
           # Install docker-compose
           DOCKER_CONFIG=${DOCKER_CONFIG:-$HOME/.docker}
@@ -202,14 +313,13 @@ jobs:
           # Run tests against it
           # using temp directory for ipfs stuff
           export IPFS_PATH=$(mktemp -d)
-          
+
           # Initialize IPFS repo
           ./ipfs init -e
-          
+
           # Copy over swarm key and config
           cp -rav $(pwd)/docker/ipfs_data/* "${IPFS_PATH}/"
-          
-          export BACALHAU_API_HOST=127.0.0.1
+
           export BACALHAU_SERVE_IPFS_PATH="${IPFS_PATH}"
           export BACALHAU_IPFS_SWARM_ADDRESSES="/ip4/127.0.0.1/tcp/4001/p2p/12D3KooWLpoHJCGxxKozRaUK1e1m2ocyVPB9dzbsU2cydujYBCD7"
 
@@ -225,10 +335,21 @@ jobs:
             find . -name '*docked.sdf' | grep 'docked.sdf'
           fi
 
+      - name: Run bacalhau describe
+        # run always even when
+        if: always()
+        run: |
+          # Execute permission
+          chmod +x ./bacalhau
+          JOB_ID=$(./bacalhau list --output json | jq -r '.[0].State.JobID')
+          ./bacalhau describe ${JOB_ID}
+
       - name: upload outputs
+        # run always even when
+        if: always()
         uses: actions/upload-artifact@v3
         with:
-          name: ci-private-output
+          name: ci-private-output-${{matrix.os}}
           path: |
             job-*
             jobs/
diff --git a/README.md b/README.md
@@ -137,15 +137,16 @@ docker compose -f docker-compose.yml -f docker-compose.private.yml  up -d
 To run `plex` cli against local private environment `export` the following params to your shell before executing `plex` commands:
 ```
 
-# using temp directory for ipfs stuff
-IPFS_PATH=$(mktemp -d)
+# using temp directory for ipfs repo
+export IPFS_PATH=$(mktemp -d)
 
 # Initialize IPFS repo
 ipfs init -e
 
 # Copy over swarm key and config
 cp -rav $(pwd)/docker/ipfs_data/* "${IPFS_PATH}/"
 
+export BACALHAU_API_HOST="127.0.0.1"
 export BACALHAU_SERVE_IPFS_PATH="${IPFS_PATH}"
 export BACALHAU_IPFS_SWARM_ADDRESSES="/ip4/127.0.0.1/tcp/4001/p2p/12D3KooWLpoHJCGxxKozRaUK1e1m2ocyVPB9dzbsU2cydujYBCD7"
 ```

diff --git a/docker/images/ipfs/Dockerfile b/docker/images/ipfs/Dockerfile
@@ -1,33 +1,4 @@
 ARG IPFS_VERSION=0.18.0
-FROM golang:1.19.1-buster AS builder
-
-ARG IPFS_VERSION=0.18.0
-ARG IPFS_DS_S3_PLUGIN_VERSION=latest
-ARG IPFS_HEALTHCHECK_PLUGIN_VERSION=latest
-
-WORKDIR /
-
-# Kubo build process
-# See details: https://github.com/ipfs/go-ds-s3
-ENV GO111MODULE on
-ENV GOPROXY direct
-
-# We clone Kubo source code
-RUN git clone --depth 1 -b v${IPFS_VERSION} https://github.com/ipfs/kubo
-ENV SRC_DIR /kubo
-
-# Move to kubo folder
-WORKDIR $SRC_DIR
-
-# Install the plugin and build ipfs
-RUN go get github.com/ceramicnetwork/go-ipfs-healthcheck/plugin@${IPFS_HEALTHCHECK_PLUGIN_VERSION}
-RUN echo "\nhealthcheck github.com/ceramicnetwork/go-ipfs-healthcheck/plugin 0" >> plugin/loader/preload_list
-RUN go get github.com/3box/go-ds-s3/plugin@${IPFS_DS_S3_PLUGIN_VERSION}
-RUN echo "\ns3ds github.com/3box/go-ds-s3/plugin 0" >> plugin/loader/preload_list
-RUN make build
-RUN go mod tidy
-RUN CGO_ENABLED=0 make build
-RUN make install
 
 # IPFS container with all the utilities
 FROM ipfs/kubo:v${IPFS_VERSION} AS ipfs-with-utils
@@ -45,13 +16,40 @@ COPY container-init.d container-init.d
 RUN chown ipfs: /container-init.d/*.sh
 RUN chown ipfs: /data/ipfs
 
+# Copy custom IPFS binary with s3ds and healthcheck plugin
+COPY --from=ceramicnetwork/go-ipfs-daemon:daaf07a9dbd007a3a6378f34f2a404992ade5e93 /usr/local/bin/ipfs /usr/local/bin/ipfs
+
 USER ipfs
 
-# Final IPFS container with custom IPFS binary and utils
-FROM ipfs-with-utils
-ENV SRC_DIR /kubo
+FROM scratch
 
-# We copy the new binaries we built in the 'builder' stage (--from=builder)
-COPY --from=builder $SRC_DIR/cmd/ipfs/ipfs /usr/local/bin/ipfs
+COPY --from=ipfs-with-utils / /
 
-USER ipfs
+# Swarm TCP; should be exposed to the public
+EXPOSE 4001
+# Swarm UDP; should be exposed to the public
+EXPOSE 4001/udp
+# Daemon API; must not be exposed publicly but to client services under you control
+EXPOSE 5001
+# Web Gateway; can be exposed publicly with a proxy, e.g. as https://ipfs.example.org
+EXPOSE 8080
+# Swarm Websockets; must be exposed publicly when the node is listening using the websocket transport (/ipX/.../tcp/8081/ws).
+EXPOSE 8081
+
+ENV IPFS_PATH /data/ipfs
+
+# The default logging level
+ENV IPFS_LOGGING ""
+
+# This just makes sure that:
+# 1. There's an fs-repo, and initializes one if there isn't.
+# 2. The API and Gateway are accessible from outside the container.
+ENTRYPOINT ["/sbin/tini", "--", "/usr/local/bin/start_ipfs"]
+
+# Healthcheck for the container
+# QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn is the CID of empty folder
+HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \
+  CMD ipfs dag stat /ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn || exit 1
+
+# Execute the daemon subcommand by default
+CMD ["daemon", "--migrate=true", "--agent-version-suffix=docker"]