Skip to content

Commit

Permalink
ci: Separate build for tensorflow
Browse files Browse the repository at this point in the history
  • Loading branch information
jailuthra committed Jun 22, 2021
1 parent d7b9d9c commit 12004bb
Show file tree
Hide file tree
Showing 3 changed files with 105 additions and 17 deletions.
102 changes: 102 additions & 0 deletions .github/workflows/linux-tensorflow.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
name: Linux Tensorflow Build
on:
pull_request:
push:
branches:
- master
jobs:
build:
runs-on: ubuntu-18.04
steps:
- name: Checkout
uses: actions/checkout@v2
with:
# Needed for commands that depend on git tags
fetch-depth: 0
ref: ${{ github.event.pull_request.head.sha }}
- name: Setup Go
uses: actions/setup-go@v2
with:
go-version: '1.15.5'
- name: Cache go modules
id: cache-go-mod
uses: actions/cache@v2.1.5
with:
path: ~/go/pkg/mod
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-
- name: Cache ffmpeg
id: cache-ffmpeg
uses: actions/cache@v2.1.5
with:
path: ~/compiled
key: ${{ runner.os }}-ffmpeg-tensorflow-${{ hashFiles('**/install_ffmpeg.sh') }}
restore-keys: |
${{ runner.os }}-ffmpeg-tensorflow
- name: Cache binaries
uses: actions/cache@v2.1.5
with:
path: ~/build
key: ${{ runner.os }}-binaries-tensorflow-${{ github.sha }}
- name: Install dependencies
run: |
sudo apt-get update \
&& sudo apt-get install -y software-properties-common curl apt-transport-https \
&& sudo apt-key adv --keyserver keyserver.ubuntu.com --recv 15CF4D18AF4F7421 \
&& sudo add-apt-repository "deb [arch=amd64] http://apt.llvm.org/xenial/ llvm-toolchain-xenial-8 main" \
&& sudo apt-get update \
&& sudo apt-get -y install clang-8 clang-tools-8 build-essential pkg-config autoconf gnutls-dev git python
sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-8 30 \
&& sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-8 30
LIBTENSORFLOW_VERSION=2.3.0 \
&& curl -LO https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-gpu-linux-x86_64-${LIBTENSORFLOW_VERSION}.tar.gz \
&& sudo tar -C /usr/local -xzf libtensorflow-gpu-linux-x86_64-${LIBTENSORFLOW_VERSION}.tar.gz \
&& sudo ldconfig
- name: Install go modules
if: steps.cache-go-mod.outputs.cache-hit != 'true'
run: go mod download
- name: Install ffmpeg
if: steps.cache-ffmpeg.outputs.cache-hit != 'true'
run: BUILD_TAGS=experimental ./install_ffmpeg.sh
- name: Build binaries
run: |
export PKG_CONFIG_PATH=~/compiled/lib/pkgconfig
./ci_env.sh make livepeer livepeer_cli livepeer_bench livepeer_router
rm -rf ~/build && mkdir ~/build && mv livepeer* ~/build/
env:
GHA_REF: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.ref || github.ref }}
upload:
runs-on: ubuntu-18.04
needs: [build]
if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository
steps:
- name: Checkout
uses: actions/checkout@v2
with:
# Needed for commands that depend on git tags
fetch-depth: 0
ref: ${{ github.event.pull_request.head.sha }}
- name: Install libtensorflow
run: |
LIBTENSORFLOW_VERSION=2.3.0 \
&& curl -LO https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-gpu-linux-x86_64-${LIBTENSORFLOW_VERSION}.tar.gz \
&& sudo tar -C /usr/local -xzf libtensorflow-gpu-linux-x86_64-${LIBTENSORFLOW_VERSION}.tar.gz \
&& sudo ldconfig
- name: Cache binaries
uses: actions/cache@v2.1.5
with:
path: ~/build
key: ${{ runner.os }}-binaries-tensorflow-${{ github.sha }}
- name: Upload build
run: cp ~/build/* . && ./upload_build.sh
env:
GHA_REF: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.ref || github.ref }}
GCLOUD_KEY: ${{ secrets.GCLOUD_KEY }}
GCLOUD_SECRET: ${{ secrets.GCLOUD_SECRET }}
DISCORD_URL: ${{ secrets.DISCORD_URL }}
RELEASE_TAG: 'tensorflow'
- name: Notify new build upload
run: curl -X POST https://holy-bread-207a.livepeer.workers.dev
17 changes: 0 additions & 17 deletions .github/workflows/linux.yml
Original file line number Diff line number Diff line change
Expand Up @@ -50,11 +50,6 @@ jobs:
sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-8 30 \
&& sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-8 30
LIBTENSORFLOW_VERSION=2.3.0 \
&& curl -LO https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-gpu-linux-x86_64-${LIBTENSORFLOW_VERSION}.tar.gz \
&& sudo tar -C /usr/local -xzf libtensorflow-gpu-linux-x86_64-${LIBTENSORFLOW_VERSION}.tar.gz \
&& sudo ldconfig
- name: Install go modules
if: steps.cache-go-mod.outputs.cache-hit != 'true'
run: go mod download
Expand All @@ -80,12 +75,6 @@ jobs:
uses: actions/setup-go@v2
with:
go-version: '1.15.5'
- name: Install libtensorflow
run: |
LIBTENSORFLOW_VERSION=2.3.0 \
&& curl -LO https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-gpu-linux-x86_64-${LIBTENSORFLOW_VERSION}.tar.gz \
&& sudo tar -C /usr/local -xzf libtensorflow-gpu-linux-x86_64-${LIBTENSORFLOW_VERSION}.tar.gz \
&& sudo ldconfig
- name: Cache go modules
id: cache-go-mod
uses: actions/cache@v2.1.5
Expand Down Expand Up @@ -123,12 +112,6 @@ jobs:
# Needed for commands that depend on git tags
fetch-depth: 0
ref: ${{ github.event.pull_request.head.sha }}
- name: Install libtensorflow
run: |
LIBTENSORFLOW_VERSION=2.3.0 \
&& curl -LO https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-gpu-linux-x86_64-${LIBTENSORFLOW_VERSION}.tar.gz \
&& sudo tar -C /usr/local -xzf libtensorflow-gpu-linux-x86_64-${LIBTENSORFLOW_VERSION}.tar.gz \
&& sudo ldconfig
- name: Cache binaries
uses: actions/cache@v2.1.5
with:
Expand Down
3 changes: 3 additions & 0 deletions upload_build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,9 @@ if [[ $(uname) == *"MSYS"* ]]; then
else
ARCH=$(uname | tr '[:upper:]' '[:lower:]')
EXT=""
if [[ -n "${RELEASE_TAG:-}" ]]; then
ARCH="$ARCH-$RELEASE_TAG"
fi
fi

BASE="livepeer-$ARCH-amd64"
Expand Down

0 comments on commit 12004bb

Please sign in to comment.