From 0a7d55cae6170f25d3020b2400663da2fad05957 Mon Sep 17 00:00:00 2001 From: Abhishek Arya Date: Mon, 13 Apr 2020 19:20:00 -0700 Subject: [PATCH 1/6] Switch OSS projects to use native go-fuzz. --- projects/go-attestation/build.sh | 2 +- projects/go-dns/build.sh | 2 +- projects/go-json-iterator/build.sh | 2 +- projects/golang/build.sh | 2 +- projects/gonids/build.sh | 2 +- projects/kubernetes/Dockerfile | 1 - projects/kubernetes/build.sh | 2 +- projects/syzkaller/build.sh | 2 +- 8 files changed, 7 insertions(+), 8 deletions(-) diff --git a/projects/go-attestation/build.sh b/projects/go-attestation/build.sh index 1aec57286bce..92d6de56c20c 100755 --- a/projects/go-attestation/build.sh +++ b/projects/go-attestation/build.sh @@ -22,7 +22,7 @@ function compile_fuzzer { fuzzer=$3 # Instrument all Go files relevant to this fuzzer - go-fuzz-build -libfuzzer -func $function -o $fuzzer.a $package + go-fuzz -func $function -o $fuzzer.a $package # Instrumented, compiled Go ($fuzzer.a) + fuzzing engine = fuzzer binary $CXX $CXXFLAGS $LIB_FUZZING_ENGINE $fuzzer.a -lpthread -o $OUT/$fuzzer diff --git a/projects/go-dns/build.sh b/projects/go-dns/build.sh index 26be8083ed7a..e2c3f29ffe99 100755 --- a/projects/go-dns/build.sh +++ b/projects/go-dns/build.sh @@ -22,7 +22,7 @@ function compile_fuzzer { fuzzer=$3 # Instrument all Go files relevant to this fuzzer - go-fuzz-build -tags fuzz -libfuzzer -func $function -o $fuzzer.a $path + go-fuzz -tags fuzz -func $function -o $fuzzer.a $path # Instrumented, compiled Go ($fuzzer.a) + fuzzing engine = fuzzer binary $CXX $CXXFLAGS $LIB_FUZZING_ENGINE $fuzzer.a -lpthread -o $OUT/$fuzzer diff --git a/projects/go-json-iterator/build.sh b/projects/go-json-iterator/build.sh index c8edbc320e87..476de37d828b 100755 --- a/projects/go-json-iterator/build.sh +++ b/projects/go-json-iterator/build.sh @@ -22,7 +22,7 @@ function compile_fuzzer { fuzzer=$3 # Instrument all Go files relevant to this fuzzer - go-fuzz-build -libfuzzer -func $function -o $fuzzer.a $path + go-fuzz -func $function -o $fuzzer.a $path # Instrumented, compiled Go ($fuzzer.a) + fuzzing engine = fuzzer binary $CXX $CXXFLAGS $LIB_FUZZING_ENGINE $fuzzer.a -lpthread -o $OUT/$fuzzer diff --git a/projects/golang/build.sh b/projects/golang/build.sh index efcbcfa1a447..30b8ccf2e5c5 100755 --- a/projects/golang/build.sh +++ b/projects/golang/build.sh @@ -2,7 +2,7 @@ function compile_fuzzer { fuzzer=$(basename $1) # Instrument all Go files relevant to this fuzzer, compile and store in $fuzzer.a - go-fuzz-build -libfuzzer -o $fuzzer.a github.com/dvyukov/go-fuzz-corpus/$fuzzer + go-fuzz -o $fuzzer.a github.com/dvyukov/go-fuzz-corpus/$fuzzer # Instrumented, compiled Go ($fuzzer.a) + libFuzzer = fuzzer binary $CXX $CXXFLAGS $LIB_FUZZING_ENGINE $fuzzer.a -lpthread -o fuzzer-$fuzzer diff --git a/projects/gonids/build.sh b/projects/gonids/build.sh index e98d54ca68a3..5f241aaff258 100755 --- a/projects/gonids/build.sh +++ b/projects/gonids/build.sh @@ -22,7 +22,7 @@ function compile_fuzzer { fuzzer=$3 # Instrument all Go files relevant to this fuzzer - go-fuzz-build -libfuzzer -func $function -o $fuzzer.a $path + go-fuzz -func $function -o $fuzzer.a $path # Instrumented, compiled Go ($fuzzer.a) + fuzzing engine = fuzzer binary $CXX $CXXFLAGS $LIB_FUZZING_ENGINE $fuzzer.a -lpthread -o $OUT/$fuzzer diff --git a/projects/kubernetes/Dockerfile b/projects/kubernetes/Dockerfile index e0c5afda2e4e..24b650936ef9 100644 --- a/projects/kubernetes/Dockerfile +++ b/projects/kubernetes/Dockerfile @@ -2,7 +2,6 @@ FROM gcr.io/oss-fuzz-base/base-builder ENV GOPATH /gopath -RUN go get -u -d github.com/dvyukov/go-fuzz/... RUN go get github.com/ianlancetaylor/demangle RUN git clone --depth 1 https://github.com/kubernetes/kubernetes.git /gopath/src/k8s.io/kubernetes diff --git a/projects/kubernetes/build.sh b/projects/kubernetes/build.sh index 539631bf4016..ff4b0b32877f 100755 --- a/projects/kubernetes/build.sh +++ b/projects/kubernetes/build.sh @@ -27,7 +27,7 @@ function compile_fuzzer { local fuzzer="${pkg}_${function}" # Instrument all Go files relevant to this fuzzer - go-fuzz-build -libfuzzer -func "${function}" -o "${fuzzer}.a" "k8s.io/kubernetes/test/fuzz/${pkg}" + go-fuzz -func "${function}" -o "${fuzzer}.a" "k8s.io/kubernetes/test/fuzz/${pkg}" # Instrumented, compiled Go ($fuzzer.a) + fuzzing engine = fuzzer binary $CXX $CXXFLAGS $LIB_FUZZING_ENGINE "${fuzzer}.a" -lpthread -o "${OUT}/${fuzzer}" diff --git a/projects/syzkaller/build.sh b/projects/syzkaller/build.sh index 44be203f7f36..2977b577af3c 100755 --- a/projects/syzkaller/build.sh +++ b/projects/syzkaller/build.sh @@ -22,7 +22,7 @@ function compile_fuzzer { fuzzer=$3 # Instrument all Go files relevant to this fuzzer - go-fuzz-build -libfuzzer -func $function -o $fuzzer.a $path + go-fuzz -func $function -o $fuzzer.a $path # Instrumented, compiled Go ($fuzzer.a) + fuzzing engine = fuzzer binary $CXX $CXXFLAGS $LIB_FUZZING_ENGINE $fuzzer.a -lpthread -o $OUT/$fuzzer From b69c2ec78f57334fd5d5740b56b19af2a8bfcfaa Mon Sep 17 00:00:00 2001 From: Abhishek Arya Date: Mon, 13 Apr 2020 19:22:13 -0700 Subject: [PATCH 2/6] Fix license headers. --- projects/golang/build.sh | 14 ++++++++++++++ projects/kubernetes/Dockerfile | 17 ++++++++++++++++- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/projects/golang/build.sh b/projects/golang/build.sh index 30b8ccf2e5c5..147192ebdbbe 100755 --- a/projects/golang/build.sh +++ b/projects/golang/build.sh @@ -1,3 +1,17 @@ +# Copyright 2020 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + function compile_fuzzer { fuzzer=$(basename $1) diff --git a/projects/kubernetes/Dockerfile b/projects/kubernetes/Dockerfile index 24b650936ef9..8c095d29e4d0 100644 --- a/projects/kubernetes/Dockerfile +++ b/projects/kubernetes/Dockerfile @@ -1,9 +1,24 @@ +# Copyright 2020 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ + FROM gcr.io/oss-fuzz-base/base-builder ENV GOPATH /gopath RUN go get github.com/ianlancetaylor/demangle - RUN git clone --depth 1 https://github.com/kubernetes/kubernetes.git /gopath/src/k8s.io/kubernetes WORKDIR /gopath/src/k8s.io/kubernetes From 6a24a3634125d4583c4869bc3529dd3e095e5c50 Mon Sep 17 00:00:00 2001 From: Abhishek Arya Date: Mon, 13 Apr 2020 19:52:26 -0700 Subject: [PATCH 3/6] Fix documentation. --- docs/getting-started/new-project-guide/go_lang.md | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/docs/getting-started/new-project-guide/go_lang.md b/docs/getting-started/new-project-guide/go_lang.md index 939bae739bb7..0965842f1255 100644 --- a/docs/getting-started/new-project-guide/go_lang.md +++ b/docs/getting-started/new-project-guide/go_lang.md @@ -22,13 +22,18 @@ process. The key specifics of integrating a Go project are outlined below. ## Go-fuzz support OSS-Fuzz supports **go-fuzz** in the -[libFuzzer compatible mode](https://github.com/dvyukov/go-fuzz#libfuzzer-support) +[libFuzzer compatible mode](https://github.com/mdempsky/go114-fuzz-build) only. In that mode, fuzz targets for Go use the libFuzzer engine with native Go coverage instrumentation. Binaries compiled in this mode provide the same libFuzzer command line interface as non-Go fuzz targets. ## Project files +First, you need to write a Go fuzz target that accepts a stream of +random bytes and calls the program API with that. This fuzz target +should reside in your project repository +([example](https://github.com/golang/go/blob/4ad13555184eb0697c2e92c64c1b0bdb287ccc10/src/html/fuzz.go#L13)). + The structure of the project directory in OSS-Fuzz repository doesn't differ for projects written in Go. The project files have the following Go specific aspects. @@ -66,7 +71,7 @@ RUN go get github.com/ianlancetaylor/demangle ### build.sh -In order to build a Go fuzz target, you need to call `go-fuzz-build -libfuzzer` +In order to build a Go fuzz target, you need to call `go-fuzz` command first, and then link the resulting `.a` file against `$LIB_FUZZING_ENGINE` using the `$CXX $CXXFLAGS ...` command. [Example](https://github.com/google/oss-fuzz/blob/356f2b947670b7eb33a1f535c71bc5c87a60b0d1/projects/syzkaller/build.sh#L19): @@ -78,7 +83,7 @@ function compile_fuzzer { fuzzer=$3 # Instrument all Go files relevant to this fuzzer - go-fuzz-build -libfuzzer -func $function -o $fuzzer.a $path + go-fuzz -func $function -o $fuzzer.a $path # Instrumented, compiled Go ($fuzzer.a) + fuzzing engine = fuzzer binary $CXX $CXXFLAGS $LIB_FUZZING_ENGINE $fuzzer.a -lpthread -o $OUT/$fuzzer From 0828cb037919acfa48afac4d6fffb21bd283622e Mon Sep 17 00:00:00 2001 From: Abhishek Arya Date: Mon, 13 Apr 2020 20:39:19 -0700 Subject: [PATCH 4/6] Fix go-json-iterator breakage, put source in package search dir. --- projects/go-json-iterator/Dockerfile | 6 +++--- projects/go-json-iterator/build.sh | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/projects/go-json-iterator/Dockerfile b/projects/go-json-iterator/Dockerfile index eef259a0a943..57cac6d56ff0 100644 --- a/projects/go-json-iterator/Dockerfile +++ b/projects/go-json-iterator/Dockerfile @@ -18,7 +18,7 @@ FROM gcr.io/oss-fuzz-base/base-builder MAINTAINER taowen@gmail.com RUN go get github.com/json-iterator/go -RUN mkdir fuzz -COPY fuzz_json.go fuzz/ +RUN mkdir $GOPATH/src/fuzz +COPY fuzz_json.go $GOPATH/src/fuzz COPY build.sh $SRC/ -WORKDIR fuzz +WORKDIR $GOPATH/src/fuzz diff --git a/projects/go-json-iterator/build.sh b/projects/go-json-iterator/build.sh index 476de37d828b..a6b1212218bf 100755 --- a/projects/go-json-iterator/build.sh +++ b/projects/go-json-iterator/build.sh @@ -28,4 +28,4 @@ function compile_fuzzer { $CXX $CXXFLAGS $LIB_FUZZING_ENGINE $fuzzer.a -lpthread -o $OUT/$fuzzer } -compile_fuzzer . Fuzz fuzz_json +compile_fuzzer fuzz Fuzz fuzz_json From bf94c6271577c51a7cd93182ac169fd0bcc27af1 Mon Sep 17 00:00:00 2001 From: Abhishek Arya Date: Mon, 13 Apr 2020 20:55:24 -0700 Subject: [PATCH 5/6] Remove random. --- docs/getting-started/new-project-guide/go_lang.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/getting-started/new-project-guide/go_lang.md b/docs/getting-started/new-project-guide/go_lang.md index 0965842f1255..f58716dfc1b7 100644 --- a/docs/getting-started/new-project-guide/go_lang.md +++ b/docs/getting-started/new-project-guide/go_lang.md @@ -29,9 +29,9 @@ libFuzzer command line interface as non-Go fuzz targets. ## Project files -First, you need to write a Go fuzz target that accepts a stream of -random bytes and calls the program API with that. This fuzz target -should reside in your project repository +First, you need to write a Go fuzz target that accepts a stream of bytes and +calls the program API with that. This fuzz target should reside in your project +repository ([example](https://github.com/golang/go/blob/4ad13555184eb0697c2e92c64c1b0bdb287ccc10/src/html/fuzz.go#L13)). The structure of the project directory in OSS-Fuzz repository doesn't differ for From 319eb42ff2c0c908e88a31aa680cb01aa0d47573 Mon Sep 17 00:00:00 2001 From: Abhishek Arya Date: Mon, 13 Apr 2020 21:29:44 -0700 Subject: [PATCH 6/6] Revert syzkaller change, track bug in #3639 --- projects/syzkaller/Dockerfile | 4 ++-- projects/syzkaller/build.sh | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/projects/syzkaller/Dockerfile b/projects/syzkaller/Dockerfile index 77bfde642730..a02296570f90 100644 --- a/projects/syzkaller/Dockerfile +++ b/projects/syzkaller/Dockerfile @@ -22,6 +22,6 @@ RUN go get -u -d github.com/google/syzkaller/... # Dependency for one of the fuzz targets. RUN go get github.com/ianlancetaylor/demangle -RUN git clone --depth 1 https://github.com/google/syzkaller.git syzkaller -WORKDIR syzkaller +RUN git clone --depth 1 https://github.com/google/syzkaller.git $GOPATH/src/syzkaller +WORKDIR $GOPATH/src/syzkaller COPY build.sh $SRC/ diff --git a/projects/syzkaller/build.sh b/projects/syzkaller/build.sh index 2977b577af3c..b36f446d244d 100755 --- a/projects/syzkaller/build.sh +++ b/projects/syzkaller/build.sh @@ -22,16 +22,16 @@ function compile_fuzzer { fuzzer=$3 # Instrument all Go files relevant to this fuzzer - go-fuzz -func $function -o $fuzzer.a $path + go-fuzz-build -libfuzzer -func $function -o $fuzzer.a $path # Instrumented, compiled Go ($fuzzer.a) + fuzzing engine = fuzzer binary $CXX $CXXFLAGS $LIB_FUZZING_ENGINE $fuzzer.a -lpthread -o $OUT/$fuzzer } -compile_fuzzer ./pkg/compiler Fuzz compiler_fuzzer -compile_fuzzer ./prog/test FuzzDeserialize prog_deserialize_fuzzer -compile_fuzzer ./prog/test FuzzParseLog prog_parselog_fuzzer -compile_fuzzer ./pkg/report Fuzz report_fuzzer +compile_fuzzer syzkaller/pkg/compiler Fuzz compiler_fuzzer +compile_fuzzer syzkaller/prog/test FuzzDeserialize prog_deserialize_fuzzer +compile_fuzzer syzkaller/prog/test FuzzParseLog prog_parselog_fuzzer +compile_fuzzer syzkaller/pkg/report Fuzz report_fuzzer # This target is way too spammy and OOMs very quickly. # compile_fuzzer ./tools/syz-trace2syz/proggen Fuzz trace2syz_fuzzer