Skip to content

Commit

Permalink
Merge branch 'master' into av/maxsessions-set-with-t-capacity
Browse files Browse the repository at this point in the history
  • Loading branch information
@eliteprox
eliteprox committed Aug 31, 2023
2 parents 18726e3 + 5af79b0 commit e626817
Show file tree
Hide file tree
Showing 11 changed files with 129 additions and 125 deletions.
109 changes: 50 additions & 59 deletions .github/workflows/build.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,6 @@ jobs:
linux-build:
name: Build binaries for ${{ matrix.platform.name }}-${{ matrix.arch }}
runs-on: ${{ matrix.platform.runner }}
container:
image: ubuntu:18.04
strategy:
fail-fast: false
matrix:
Expand All @@ -26,9 +24,9 @@ jobs:
- amd64
platform:
- name: linux
runner: ubuntu-latest
runner: ubuntu-20.04
- name: windows
runner: ubuntu-latest
runner: ubuntu-20.04
exclude:
- platform:
name: windows
Expand All @@ -37,10 +35,10 @@ jobs:
steps:
- name: Setup ubuntu container
run: |
apt update
apt install -yqq build-essential make software-properties-common
add-apt-repository -y ppa:git-core/ppa
apt update && apt install -yqq git zip unzip zlib1g-dev zlib1g
sudo apt update
sudo apt install -yqq build-essential make software-properties-common
sudo add-apt-repository -y ppa:git-core/ppa
sudo apt update && sudo apt install -yqq git zip unzip zlib1g-dev zlib1g
- name: Check out code
uses: actions/checkout@v3.5.3
Expand All @@ -50,6 +48,10 @@ jobs:
# for ref value discussion
ref: ${{ github.event.pull_request.head.sha }}

- name: Cleanup hosted runner
run: |
sudo rm -rf /usr/share/dotnet /usr/local/lib/android
- name: Set up go
id: go
uses: actions/setup-go@v4
Expand All @@ -62,17 +64,11 @@ jobs:
id: cache-ffmpeg
uses: actions/cache@v3
with:
path: /github/home/compiled
path: /home/runner/compiled
key: ${{ runner.os }}-${{ matrix.platform.name }}-${{ matrix.arch }}-ffmpeg-${{ hashFiles('**/install_ffmpeg.sh') }}
restore-keys: |
${{ runner.os }}-${{ matrix.platform.name }}-${{ matrix.arch }}-ffmpeg-
- name: Cache binaries
uses: actions/cache@v3
with:
path: ~/build
key: ${{ runner.os }}-${{ matrix.platform.name }}-${{ matrix.arch }}-binaries-${{ github.sha }}

- name: Set build environment
run: |
echo "GOARCH=${{ matrix.arch }}" >> $GITHUB_ENV
Expand All @@ -81,16 +77,18 @@ jobs:
- name: Install dependencies
run: |
apt-get update \
&& apt-get install -y software-properties-common curl apt-transport-https \
&& curl -fsSl https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - \
&& add-apt-repository "deb [arch=amd64] http://apt.llvm.org/xenial/ llvm-toolchain-xenial-12 main" \
&& apt-get update \
&& apt-get -y install clang-12 clang-tools-12 lld-12 build-essential pkg-config autoconf git python gcc-multilib libgcc-8-dev-arm64-cross mingw-w64-tools gcc-mingw-w64-x86-64
update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-12 30 \
&& update-alternatives --install /usr/bin/clang clang /usr/bin/clang-12 30 \
&& update-alternatives --install /usr/bin/ld ld /usr/bin/lld-12 30
sudo apt update \
&& sudo apt install -yqq software-properties-common curl apt-transport-https lsb-release \
&& curl -fsSl https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - \
&& sudo add-apt-repository "deb https://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs)-12 main" \
&& sudo apt update \
&& sudo apt -yqq install \
clang-12 clang-tools-12 lld-12 build-essential pkg-config autoconf git python \
gcc-multilib gcc-mingw-w64 libgcc-9-dev-arm64-cross mingw-w64-tools gcc-mingw-w64-x86-64
sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-12 30 \
&& sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-12 30 \
&& sudo update-alternatives --install /usr/bin/ld ld /usr/bin/lld-12 30
- name: Install go modules
if: steps.go.outputs.cache-hit != 'true'
Expand All @@ -103,9 +101,9 @@ jobs:
- name: Build binaries
env:
GHA_REF: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.ref || github.ref }}
PKG_CONFIG_PATH: "/home/runner/compiled/lib/pkgconfig"
run: |
git config --global --add safe.directory '*'
export PKG_CONFIG_PATH=/github/home/compiled/lib/pkgconfig
./ci_env.sh make
- name: Upload build
Expand Down Expand Up @@ -171,12 +169,6 @@ jobs:
restore-keys: |
${{ runner.os }}-${{ matrix.platform.name }}-${{ matrix.arch }}-ffmpeg-
- name: Cache binaries
uses: actions/cache@v3
with:
path: ~/build
key: ${{ runner.os }}-${{ matrix.platform.name }}-${{ matrix.arch }}-binaries-${{ github.sha }}

- name: Install dependencies
run: brew install coreutils pkg-config

Expand Down Expand Up @@ -234,16 +226,15 @@ jobs:

linux-tensorflow:
name: Build binaries for linux using tensorflow
runs-on: ubuntu-latest
container:
image: ubuntu:18.04
runs-on: ubuntu-20.04

steps:
- name: Setup ubuntu container
run: |
apt update
apt install -yqq build-essential make software-properties-common
add-apt-repository -y ppa:git-core/ppa
apt update && apt install -yqq git zip unzip zlib1g-dev zlib1g coreutils binutils libc++-dev libc++abi-dev libtool p7zip-full mediainfo xz-utils
sudo apt update
sudo apt install -yqq build-essential make software-properties-common
sudo add-apt-repository -y ppa:git-core/ppa
sudo apt update && sudo apt install -yqq git zip unzip zlib1g-dev zlib1g coreutils binutils libc++-dev libc++abi-dev libtool p7zip-full mediainfo xz-utils
- name: Check out code
uses: actions/checkout@v3.5.3
Expand All @@ -253,6 +244,10 @@ jobs:
# for ref value discussion
ref: ${{ github.event.pull_request.head.sha }}

- name: Cleanup hosted runner
run: |
sudo rm -rf /usr/share/dotnet /usr/local/lib/android
- name: Set up go
id: go
uses: actions/setup-go@v4
Expand All @@ -265,33 +260,29 @@ jobs:
id: cache-ffmpeg
uses: actions/cache@v3
with:
path: /github/home/compiled
path: /home/runner/compiled
key: ${{ runner.os }}-ffmpeg-tf-${{ hashFiles('**/install_ffmpeg.sh') }}
restore-keys: |
${{ runner.os }}-ffmpeg-tf-
- name: Cache binaries
uses: actions/cache@v3
with:
path: ~/build
key: ${{ runner.os }}-binaries-tf-${{ github.sha }}

- name: Install dependencies
env:
LIBTENSORFLOW_VERSION: "2.12.1"
run: |
apt-get update \
&& apt-get install -y software-properties-common curl apt-transport-https \
&& curl -fsSl https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - \
&& add-apt-repository "deb [arch=amd64] http://apt.llvm.org/xenial/ llvm-toolchain-xenial-8 main" \
&& apt-get update \
&& apt-get -y install clang-8 clang-tools-8 build-essential pkg-config autoconf gnutls-dev git python
sudo apt update \
&& sudo apt install -yqq software-properties-common curl apt-transport-https lsb-release \
&& curl -fsSl https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - \
&& sudo add-apt-repository "deb https://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs)-12 main" \
&& sudo apt update \
&& sudo apt -yqq install clang-12 clang-tools-12 lld-12 build-essential pkg-config autoconf git python
update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-8 30 \
&& update-alternatives --install /usr/bin/clang clang /usr/bin/clang-8 30
sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-12 30 \
&& sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-12 30 \
&& sudo update-alternatives --install /usr/bin/ld ld /usr/bin/lld-12 30
LIBTENSORFLOW_VERSION=2.12.1 \
&& curl -LO https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-gpu-linux-x86_64-${LIBTENSORFLOW_VERSION}.tar.gz \
&& tar -C /usr/local -xzf libtensorflow-gpu-linux-x86_64-${LIBTENSORFLOW_VERSION}.tar.gz \
&& ldconfig
sudo curl -LO https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-gpu-linux-x86_64-${LIBTENSORFLOW_VERSION}.tar.gz \
&& sudo tar -C /usr/local -xzf libtensorflow-gpu-linux-x86_64-${LIBTENSORFLOW_VERSION}.tar.gz \
&& sudo ldconfig
- name: Install go modules
# if: steps.go.outputs.cache-hit != 'true'
Expand All @@ -304,9 +295,9 @@ jobs:
- name: Build binaries
env:
GHA_REF: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.ref || github.ref }}
PKG_CONFIG_PATH: "/home/runner/compiled/lib/pkgconfig"
run: |
git config --global --add safe.directory '*'
export PKG_CONFIG_PATH=/github/home/compiled/lib/pkgconfig
./ci_env.sh make cgo_cflags="-lz" all
rm -rf ~/build && mkdir ~/build && mv livepeer* ~/build/
Expand Down
53 changes: 29 additions & 24 deletions .github/workflows/test.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,16 +17,15 @@ concurrency:
jobs:
test:
name: Run tests defined for the project
runs-on: ubuntu-latest
container:
image: ubuntu:18.04
runs-on: ubuntu-20.04

steps:
- name: Setup ubuntu container
run: |
apt update
apt install -yqq build-essential make software-properties-common
add-apt-repository -y ppa:git-core/ppa
apt update && apt install -yqq git zip unzip zlib1g-dev zlib1g
sudo apt update
sudo apt install -yqq build-essential make software-properties-common
sudo add-apt-repository -y ppa:git-core/ppa
sudo apt update && sudo apt install -yqq git zip unzip zlib1g-dev zlib1g
- name: Check out code
uses: actions/checkout@v3.5.3
Expand All @@ -36,6 +35,10 @@ jobs:
# for ref value discussion
ref: ${{ github.event.pull_request.head.sha }}

- name: Cleanup hosted runner
run: |
sudo rm -rf /usr/share/dotnet /usr/local/lib/android
- name: Set up go
id: go
uses: actions/setup-go@v4
Expand All @@ -48,22 +51,29 @@ jobs:
id: cache-ffmpeg
uses: actions/cache@v3
with:
path: /github/home/compiled
path: /home/runner/compiled
key: ${{ runner.os }}-ffmpeg-${{ hashFiles('install_ffmpeg.sh') }}
restore-keys: |
${{ runner.os }}-ffmpeg
- name: Install protobuf
run: |
go install github.com/golang/protobuf/protoc-gen-go@v1.3.5
sudo apt update
sudo apt install -yqq protobuf-compiler
- name: Install dependencies
run: |
apt-get update \
&& apt-get install -yqq software-properties-common curl apt-transport-https \
&& curl -fsSl https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - \
&& add-apt-repository "deb [arch=amd64] http://apt.llvm.org/xenial/ llvm-toolchain-xenial-8 main" \
&& apt-get update \
&& apt-get -yqq install clang-8 clang-tools-8 build-essential pkg-config autoconf git python
sudo apt update \
&& sudo apt install -yqq software-properties-common curl apt-transport-https lsb-release \
&& curl -fsSl https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - \
&& sudo add-apt-repository "deb https://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs)-12 main" \
&& sudo apt update \
&& sudo apt -yqq install clang-12 clang-tools-12 lld-12 build-essential pkg-config autoconf git python
update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-8 30 \
&& update-alternatives --install /usr/bin/clang clang /usr/bin/clang-8 30
sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-12 30 \
&& sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-12 30 \
&& sudo update-alternatives --install /usr/bin/ld ld /usr/bin/lld-12 30
- name: Install go modules
# if: steps.go.outputs.cache-hit != 'true'
Expand All @@ -73,23 +83,18 @@ jobs:
if: steps.cache-ffmpeg.outputs.cache-hit != 'true'
run: ./install_ffmpeg.sh

- name: Install protobuf
run: |
go install github.com/golang/protobuf/protoc-gen-go@v1.3.5
apt-get update
apt-get install -yqq protobuf-compiler
- name: go fmt
run: |
git config --global --add safe.directory '*'
echo "PKG_CONFIG_PATH=/github/home/compiled/lib/pkgconfig" >> $GITHUB_ENV
echo "PKG_CONFIG_PATH=/home/runner/compiled/lib/pkgconfig" >> $GITHUB_ENV
go fmt ./...
git diff --exit-code
- name: Lint
uses: golangci/golangci-lint-action@v3
with:
version: v1.52.2
skip-pkg-cache: true
args: '--disable-all --enable=gofmt --enable=vet --enable=golint --deadline=4m pm verification'

- name: Run Revive Action by building from repository
Expand All @@ -103,8 +108,8 @@ jobs:
- name: Run tests with coverage
env:
CGO_CFLAGS: "-lz"
PKG_CONFIG_PATH: "/home/runner/compiled/lib/pkgconfig"
run: |
export PKG_CONFIG_PATH=/github/home/compiled/lib/pkgconfig
./test.sh
./test_e2e.sh
Expand Down
6 changes: 4 additions & 2 deletions clog/clog.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,14 +30,16 @@ const (
nonce = "nonce"
seqNo = "seqNo"
orchSessionID = "orchSessionID" // session id generated on orchestrator for broadcaster
ethaddress = "ethaddress"
orchestrator = "orchestrator"
)

// Verbose is a boolean type that implements Infof (like Printf) etc.
type Verbose bool

var stdKeys map[string]bool
var stdKeysOrder = []string{manifestID, sessionID, nonce, seqNo, orchSessionID}
var publicLogKeys = []string{manifestID, sessionID, orchSessionID, ClientIP}
var stdKeysOrder = []string{manifestID, sessionID, nonce, seqNo, orchSessionID, ethaddress, orchestrator}
var publicLogKeys = []string{manifestID, sessionID, orchSessionID, ClientIP, seqNo, ethaddress, orchestrator}

func init() {
stdKeys = make(map[string]bool)
Expand Down
22 changes: 15 additions & 7 deletions clog/clog_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,16 +15,18 @@ func TestStdKeys(t *testing.T) {
ctx = AddNonce(ctx, 1038)
ctx = AddOrchSessionID(ctx, "orchID")
ctx = AddSeqNo(ctx, 9427)
ctx = AddVal(ctx, "ethaddress", "0x0")
ctx = AddVal(ctx, "orchestrator", "http://127.0.0.1:8935")
ctx = AddVal(ctx, "customKey", "customVal")
msg, _ := formatMessage(ctx, false, false, "testing message num=%d", 452)
assert.Equal("manifestID=manID sessionID=sessionID nonce=1038 seqNo=9427 orchSessionID=orchID customKey=customVal testing message num=452", msg)
assert.Equal("manifestID=manID sessionID=sessionID nonce=1038 seqNo=9427 orchSessionID=orchID ethaddress=0x0 orchestrator=http://127.0.0.1:8935 customKey=customVal testing message num=452", msg)
ctxCloned := Clone(context.Background(), ctx)
ctxCloned = AddManifestID(ctxCloned, "newManifest")
msgCloned, _ := formatMessage(ctxCloned, false, false, "testing message num=%d", 4521)
assert.Equal("manifestID=newManifest sessionID=sessionID nonce=1038 seqNo=9427 orchSessionID=orchID customKey=customVal testing message num=4521", msgCloned)
assert.Equal("manifestID=newManifest sessionID=sessionID nonce=1038 seqNo=9427 orchSessionID=orchID ethaddress=0x0 orchestrator=http://127.0.0.1:8935 customKey=customVal testing message num=4521", msgCloned)
// old context shouldn't change
msg, _ = formatMessage(ctx, false, false, "testing message num=%d", 452)
assert.Equal("manifestID=manID sessionID=sessionID nonce=1038 seqNo=9427 orchSessionID=orchID customKey=customVal testing message num=452", msg)
assert.Equal("manifestID=manID sessionID=sessionID nonce=1038 seqNo=9427 orchSessionID=orchID ethaddress=0x0 orchestrator=http://127.0.0.1:8935 customKey=customVal testing message num=452", msg)
}

func TestLastErr(t *testing.T) {
Expand All @@ -47,9 +49,11 @@ func TestPublicLogs(t *testing.T) {
ctx := AddManifestID(context.Background(), "fooManID")
ctx = AddSessionID(ctx, "fooSessionID")
ctx = AddOrchSessionID(ctx, "fooOrchID")
ctx = AddSeqNo(ctx, 555)
ctx = AddVal(ctx, "ethaddress", "0x0")
ctx = AddVal(ctx, "orchestrator", "http://127.0.0.1:8935")
// These should not be visible:
ctx = AddNonce(ctx, 999)
ctx = AddSeqNo(ctx, 555)
ctx = AddVal(ctx, "foo", "Bar")

publicCtx := PublicCloneCtx(ctx, context.Background(), publicLogKeys)
Expand All @@ -61,16 +65,20 @@ func TestPublicLogs(t *testing.T) {
assert.Equal("fooSessionID", val)
val = GetVal(publicCtx, orchSessionID)
assert.Equal("fooOrchID", val)
val = GetVal(publicCtx, seqNo)
assert.Equal("555", val)
val = GetVal(publicCtx, "ethaddress")
assert.Equal("0x0", val)
val = GetVal(publicCtx, "orchestrator")
assert.Equal("http://127.0.0.1:8935", val)

// Verify random keys cannot be leaked:
val = GetVal(publicCtx, nonce)
assert.Equal("", val)
val = GetVal(publicCtx, seqNo)
assert.Equal("", val)
val = GetVal(publicCtx, "foo")
assert.Equal("", val)

// Verify [PublicLogs] gets pre-pended:
msg, _ := formatMessage(ctx, false, true, "testing message num=%d", 123)
assert.Equal("[PublicLogs] manifestID=fooManID sessionID=fooSessionID nonce=999 seqNo=555 orchSessionID=fooOrchID foo=Bar testing message num=123", msg)
assert.Equal("[PublicLogs] manifestID=fooManID sessionID=fooSessionID nonce=999 seqNo=555 orchSessionID=fooOrchID ethaddress=0x0 orchestrator=http://127.0.0.1:8935 foo=Bar testing message num=123", msg)
}

0 comments on commit e626817

Please sign in to comment.