Fix short line filtering (#2)

vearutop · Sep 15, 2022 · e4ec30f · e4ec30f
1 parent 8b24357
commit e4ec30f
Show file tree

Hide file tree

Showing 5 changed files with 277 additions and 0 deletions.
diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
@@ -0,0 +1,119 @@
+# This script is provided by github.com/bool64/dev.
+name: bench
+on:
+  pull_request:
+  workflow_dispatch:
+    inputs:
+      old:
+        description: 'Old Ref'
+        required: false
+        default: 'master'
+      new:
+        description: 'New Ref'
+        required: true
+
+# Cancel the workflow in progress in newer build is about to start.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+env:
+  GO111MODULE: "on"
+  CACHE_BENCHMARK: "off"    # Enables benchmark result reuse between runs, may skew latency results.
+  RUN_BASE_BENCHMARK: "on"  # Runs benchmark for PR base in case benchmark result is missing.
+  GO_VERSION: 1.19.x
+jobs:
+  bench:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install Go stable
+        if: env.GO_VERSION != 'tip'
+        uses: actions/setup-go@v3
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Install Go tip
+        if: env.GO_VERSION == 'tip'
+        run: |
+          curl -sL https://storage.googleapis.com/go-build-snap/go/linux-amd64/$(git ls-remote https://github.com/golang/go.git HEAD | awk '{print $1;}').tar.gz -o gotip.tar.gz
+          ls -lah gotip.tar.gz
+          mkdir -p ~/sdk/gotip
+          tar -C ~/sdk/gotip -xzf gotip.tar.gz
+          ~/sdk/gotip/bin/go version
+          echo "PATH=$HOME/go/bin:$HOME/sdk/gotip/bin/:$PATH" >> $GITHUB_ENV
+      - name: Checkout code
+        uses: actions/checkout@v2
+        with:
+          ref: ${{ (github.event.inputs.new != '') && github.event.inputs.new || github.event.ref }}
+      - name: Go cache
+        uses: actions/cache@v2
+        with:
+          # In order:
+          # * Module download cache
+          # * Build cache (Linux)
+          path: |
+            ~/go/pkg/mod
+            ~/.cache/go-build
+          key: ${{ runner.os }}-go-cache-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-cache
+      - name: Restore benchstat
+        uses: actions/cache@v2
+        with:
+          path: ~/go/bin/benchstat
+          key: ${{ runner.os }}-benchstat
+      - name: Restore base benchmark result
+        if: env.CACHE_BENCHMARK == 'on'
+        id: benchmark-base
+        uses: actions/cache@v2
+        with:
+          path: |
+            bench-master.txt
+            bench-main.txt
+          # Use base sha for PR or new commit hash for master/main push in benchmark result key.
+          key: ${{ runner.os }}-bench-${{ (github.event.pull_request.base.sha != github.event.after) && github.event.pull_request.base.sha || github.event.after }}
+      - name: Checkout base code
+        if: env.RUN_BASE_BENCHMARK == 'on' && steps.benchmark-base.outputs.cache-hit != 'true' && (github.event.pull_request.base.sha != '' || github.event.inputs.old != '')
+        uses: actions/checkout@v2
+        with:
+          ref: ${{ (github.event.pull_request.base.sha != '' ) && github.event.pull_request.base.sha || github.event.inputs.old }}
+          path: __base
+      - name: Run base benchmark
+        if: env.RUN_BASE_BENCHMARK == 'on' && steps.benchmark-base.outputs.cache-hit != 'true' && (github.event.pull_request.base.sha != '' || github.event.inputs.old != '')
+        run: |
+          export REF_NAME=master
+          cd __base
+          make | grep bench-run && (BENCH_COUNT=5 make bench-run bench-stat && cp bench-master.txt ../bench-master.txt) || echo "No benchmarks in base"
+      - name: Benchmark
+        id: bench
+        run: |
+          export REF_NAME=new
+          BENCH_COUNT=5 make bench
+          OUTPUT=$(make bench-stat-diff)
+          echo "${OUTPUT}"
+          OUTPUT="${OUTPUT//$'\n'/%0A}"
+          echo "::set-output name=diff::$OUTPUT"
+          OUTPUT=$(make bench-stat)
+          echo "${OUTPUT}"
+          OUTPUT="${OUTPUT//$'\n'/%0A}"
+          echo "::set-output name=result::$OUTPUT"
+      - name: Comment Benchmark Result
+        continue-on-error: true
+        uses: marocchino/sticky-pull-request-comment@v2
+        with:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          header: bench
+          message: |
+            ### Benchmark Result
+            <details><summary>Benchmark diff with base branch</summary>
+
+            ```
+            ${{ steps.bench.outputs.diff }}
+            ```
+            </details>
+
+            <details><summary>Benchmark result</summary>
+
+            ```
+            ${{ steps.bench.outputs.result }}
+            ```
+            </details>
diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml
@@ -0,0 +1,138 @@
+# This script is provided by github.com/bool64/dev.
+name: test-unit
+on:
+  push:
+    branches:
+      - master
+      - main
+  pull_request:
+
+# Cancel the workflow in progress in newer build is about to start.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+env:
+  GO111MODULE: "on"
+  RUN_BASE_COVERAGE: "on"  # Runs test for PR base in case base test coverage is missing.
+  COV_GO_VERSION: 1.18.x   # Version of Go to collect coverage
+  TARGET_DELTA_COV: 90     # Target coverage of changed lines, in percents
+jobs:
+  test:
+    strategy:
+      matrix:
+        go-version: [ 1.19.x ]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install Go stable
+        if: matrix.go-version != 'tip'
+        uses: actions/setup-go@v3
+        with:
+          go-version: ${{ matrix.go-version }}
+
+      - name: Install Go tip
+        if: matrix.go-version == 'tip'
+        run: |
+          curl -sL https://storage.googleapis.com/go-build-snap/go/linux-amd64/$(git ls-remote https://github.com/golang/go.git HEAD | awk '{print $1;}').tar.gz -o gotip.tar.gz
+          ls -lah gotip.tar.gz
+          mkdir -p ~/sdk/gotip
+          tar -C ~/sdk/gotip -xzf gotip.tar.gz
+          ~/sdk/gotip/bin/go version
+          echo "PATH=$HOME/go/bin:$HOME/sdk/gotip/bin/:$PATH" >> $GITHUB_ENV
+
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Go cache
+        uses: actions/cache@v2
+        with:
+          # In order:
+          # * Module download cache
+          # * Build cache (Linux)
+          path: |
+            ~/go/pkg/mod
+            ~/.cache/go-build
+          key: ${{ runner.os }}-go-cache-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-cache
+
+      - name: Restore base test coverage
+        id: base-coverage
+        if: matrix.go-version == env.COV_GO_VERSION && github.event.pull_request.base.sha != ''
+        uses: actions/cache@v2
+        with:
+          path: |
+            unit-base.txt
+          # Use base sha for PR or new commit hash for master/main push in test result key.
+          key: ${{ runner.os }}-unit-test-coverage-${{ (github.event.pull_request.base.sha != github.event.after) && github.event.pull_request.base.sha || github.event.after }}
+
+      - name: Run test for base code
+        if: matrix.go-version == env.COV_GO_VERSION && env.RUN_BASE_COVERAGE == 'on' && steps.base-coverage.outputs.cache-hit != 'true' && github.event.pull_request.base.sha != ''
+        run: |
+          git fetch origin master ${{ github.event.pull_request.base.sha }}
+          HEAD=$(git rev-parse HEAD)
+          git reset --hard ${{ github.event.pull_request.base.sha }}
+          (make test-unit && go tool cover -func=./unit.coverprofile > unit-base.txt) || echo "No test-unit in base"
+          git reset --hard $HEAD
+
+      - name: Test
+        id: test
+        run: |
+          make test-unit
+          go tool cover -func=./unit.coverprofile > unit.txt
+          TOTAL=$(grep 'total:' unit.txt)
+          echo "${TOTAL}"
+          echo "::set-output name=total::$TOTAL"
+
+      - name: Annotate missing test coverage
+        id: annotate
+        if: matrix.go-version == env.COV_GO_VERSION && github.event.pull_request.base.sha != ''
+        run: |
+          curl -sLO https://github.com/vearutop/gocovdiff/releases/download/v1.3.6/linux_amd64.tar.gz && tar xf linux_amd64.tar.gz
+          gocovdiff_hash=$(git hash-object ./gocovdiff)
+          [ "$gocovdiff_hash" == "8e507e0d671d4d6dfb3612309b72b163492f28eb" ] || (echo "::error::unexpected hash for gocovdiff, possible tampering: $gocovdiff_hash" && exit 1)
+          git fetch origin master ${{ github.event.pull_request.base.sha }}
+          REP=$(./gocovdiff -cov unit.coverprofile -gha-annotations gha-unit.txt -delta-cov-file delta-cov-unit.txt -target-delta-cov ${TARGET_DELTA_COV})
+          echo "${REP}"
+          REP="${REP//$'\n'/%0A}"
+          cat gha-unit.txt
+          DIFF=$(test -e unit-base.txt && ./gocovdiff -func-cov unit.txt -func-base-cov unit-base.txt || echo "Missing base coverage file")
+          DIFF="${DIFF//$'\n'/%0A}"
+          TOTAL=$(cat delta-cov-unit.txt)
+          echo "::set-output name=rep::$REP"
+          echo "::set-output name=diff::$DIFF"
+          echo "::set-output name=total::$TOTAL"
+
+      - name: Comment Test Coverage
+        continue-on-error: true
+        if: matrix.go-version == env.COV_GO_VERSION && github.event.pull_request.base.sha != ''
+        uses: marocchino/sticky-pull-request-comment@v2
+        with:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          header: unit-test
+          message: |
+            ### Unit Test Coverage
+            ${{ steps.test.outputs.total }}
+            ${{ steps.annotate.outputs.total }}
+            <details><summary>Coverage of changed lines</summary>
+            
+            ${{ steps.annotate.outputs.rep }}
+
+            </details>
+
+            <details><summary>Coverage diff with base branch</summary>
+
+            ${{ steps.annotate.outputs.diff }}
+            
+            </details>
+
+      - name: Store base coverage
+        if: ${{ github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main' }}
+        run: cp unit.txt unit-base.txt
+
+      - name: Upload code coverage
+        if: matrix.go-version == env.COV_GO_VERSION
+        uses: codecov/codecov-action@v1
+        with:
+          file: ./unit.coverprofile
+          flags: unittests
diff --git a/Makefile b/Makefile
@@ -30,6 +30,7 @@ endif
 -include $(DEVGO_PATH)/makefiles/main.mk
 -include $(DEVGO_PATH)/makefiles/lint.mk
 -include $(DEVGO_PATH)/makefiles/test-unit.mk
+-include $(DEVGO_PATH)/makefiles/bench.mk
 -include $(DEVGO_PATH)/makefiles/reset-ci.mk
 -include $(DEVGO_PATH)/makefiles/build.mk
 -include $(DEVGO_PATH)/makefiles/release-assets.mk

diff --git a/filter_test.go b/filter_test.go
@@ -28,6 +28,16 @@ func TestAlphanum(t *testing.T) {
 	}
 }
 
+func TestShortLine(t *testing.T) {
+	line := "foo-bar-12 i3 2022/09/15 11:24:10.689412 baz 0-275 foo bar"
+	expected := "X X X/X/X X:X:X.X baz X foo bar"
+	filtered := string(filterAlphanumeric([]byte(line), 120))
+
+	if expected != filtered {
+		t.Fatalf("unexpected filtered: %s", filtered)
+	}
+}
+
 // BenchmarkRegex checks regex-based filtering implementation, for reference.
 // BenchmarkRegex-12                 	   76851	     15707 ns/op	     200 B/op	       5 allocs/op.
 func BenchmarkRegex(b *testing.B) {

diff --git a/fllter.go b/fllter.go
@@ -12,6 +12,7 @@ func filterAlphanumeric(data []byte, l int) []byte {
 
 	for i = 0; i < len(data); i++ {
 		c := data[i]
+
 		isAlpha := false
 
 		switch {
@@ -53,5 +54,13 @@ func filterAlphanumeric(data []byte, l int) []byte {
 		}
 	}
 
+	if wordStart >= 0 {
+		if hasDigit {
+			res = append(res, 'X')
+		} else {
+			res = append(res, data[wordStart:i]...)
+		}
+	}
+
 	return res
 }