Merge pull request #330 from lunasec-io/improve-scanner-reliability

Improve results of log4shell scanner

tools/log4shell/Dockerfile

@@ -14,7 +14,7 @@ WORKDIR /build
 COPY . /build
 COPY --from=java-build /build/hotpatch-payload/target/classes/Log4ShellHotpatch.class /build
 
-RUN go build .
+RUN go build -o log4shell . 
 
 FROM alpine
 

tools/log4shell/Makefile

@@ -0,0 +1,21 @@
+BINARY_NAME=log4shell
+LIBRARY_HASHES=log4j-library-hashes.json
+
+payload:
+	cd payloads/hotpatch-payload/ && \
+	mvn package && \
+	cd - && \
+	cp payloads/hotpatch-payload/target/classes/Log4ShellHotpatch.class Log4ShellHotpatch.class
+
+cli:
+	touch ${LIBRARY_HASHES}
+	go build -o ${BINARY_NAME} .
+
+library-hashes: cli
+	./log4shell analyze --output ${LIBRARY_HASHES} test/vulnerable-log4j2-versions/apache test/vulnerable-log4j2-versions/target/dependency
+
+build: payload cli
+	echo "built ${BINARY_NAME}"
+
+clean:
+	go clean

tools/log4shell/README.md

@@ -17,18 +17,53 @@ A CLI tool for identifying and patching the Log4Shell vulnerability.
 
 ## Usage
 
-Scan a directory for known vulnerable Log4j dependencies.
+### Scanning
+Scan directories for known vulnerable Log4j dependencies.
 
 ```shell
-log4shell scan <dir>
+$ log4shell scan  <dir1> <dir2> ...
 ```
 
+Output findings to a file in json format with `--output`.
+
+```shell
+$ log4shell scan --output findings.json <dir>
+... 
+$ cat findings.json
+{"vulnerable_libraries":[{"path":"test/vulnerable-log4j2-versions/target/dependency/log4j-core-2.0-rc1.jar","file_name":"org/apache/logging/log4j/core/lookup/JndiLookup.class","hash":"39a495034d37c7934b64a9aa686ea06b61df21aa222044cc50a47d6903ba1ca8","version_info":"log4j 2.0-rc1","cve":"CVE-2021-44228"}, ...]}
+```
+
+To output findings, as the tool discovers them, in json format, use `--json`.
+
+```shell
+$ log4shell scan --json test/vulnerable-log4j2-versions 
+{"severity":"10.0","path":"test/vulnerable-log4j2-versions/target/dependency/log4j-core-2.0-rc1.jar","fileName":"org/apache/logging/log4j/core/lookup/JndiLookup.class","versionInfo":"log4j 2.0-rc1","cve":"CVE-2021-44228","time":1639624662,"message":"identified vulnerable path"}
+...
+```
+
+Depending on what you are scanning, you might run into a wall of warnings like `"WRN unable to open archive error="zip: not a valid zip file"`.
+You can disable these by passing `--ignore-warnings`.
+
+```shell
+$ log4shell scan --ignore-warnings <dir1> <dir2> ...
+```
+
+You may exclude subdirectories while searching by using `--exclude`. This can be used multiple times in the command to
+exclude multiple subdirectories.
+
+```shell
+$ log4shell scan --exclude <subdir1> --exclude <subdir2> <dir1> <dir2>
+```
+
+### Live Patch
 Run a Live Patch server.
 
 ```shell
-log4shell livepatch
+$ log4shell livepatch
 ```
 
+Read more about how this works [here](https://www.lunasec.io/docs/blog/log4shell-live-patch/).
+
 ## Building
 
 ```
@@ -40,7 +75,7 @@ or
 
 Make sure you have Maven installed, then:
 ```
-./build-payload.sh && go build . && ./log4shell
+make build && ./log4shell
 ```
 
 ## Releases

tools/log4shell/analyze/analyze.go

@@ -0,0 +1,142 @@
+// Copyright 2021 by LunaSec (owned by Refinery Labs, Inc)
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+package analyze
+
+import (
+	"github.com/blang/semver/v4"
+	"github.com/lunasec-io/lunasec/tools/log4shell/constants"
+	"github.com/lunasec-io/lunasec/tools/log4shell/types"
+	"github.com/lunasec-io/lunasec/tools/log4shell/util"
+	"github.com/rs/zerolog/log"
+	"io"
+	"path"
+	"strings"
+)
+
+func isVersionALog4ShellVersion(semverVersion string) bool {
+	version, _ := semver.Make(semverVersion)
+
+	vulnerableRange, _ := semver.ParseRange(">=2.0.0-beta9 <=2.14.1")
+	if vulnerableRange(version) {
+		return true
+	}
+	return false
+}
+
+func isVersionACVE202145046Version(semverVersion string) bool {
+	version, _ := semver.Make(semverVersion)
+
+	vulnerableRange, _ := semver.ParseRange("=2.15.0")
+	if vulnerableRange(version) {
+		return true
+	}
+	return false
+}
+
+func isVersionACVE201917571Version(semverVersion string) bool {
+	version, _ := semver.Make(semverVersion)
+
+	vulnerableRange, _ := semver.ParseRange(">=1.2.0 <=1.2.17")
+	if vulnerableRange(version) {
+		return true
+	}
+	return false
+}
+
+func adjustMissingPatchVersion(semverVersion string) string {
+	if len(semverVersion) == 3 {
+		semverVersion += ".0"
+	}
+	if strings.HasPrefix(semverVersion, "2.0-") {
+		semverVersion = strings.Replace(semverVersion, "2.0-", "2.0.0-", -1)
+	}
+	if strings.HasPrefix(semverVersion, "1.0-") {
+		semverVersion = strings.Replace(semverVersion, "1.0-", "1.0.0-", -1)
+	}
+	return semverVersion
+}
+
+func ProcessArchiveFile(reader io.Reader, filePath, fileName string) (finding *types.Finding) {
+	_, file := path.Split(filePath)
+	version := strings.TrimSuffix(file, path.Ext(file))
+
+	// small adjustments to the version so that it can be parsed as semver
+	semverVersion := strings.Replace(version, "log4j-core-", "", -1)
+	semverVersion = strings.Replace(semverVersion, "logging-log4j-", "", -1)
+	semverVersion = strings.Replace(semverVersion, "jakarta-log4j-", "", -1)
+	semverVersion = strings.Replace(semverVersion, "log4j-", "", -1)
+
+	semverVersion = adjustMissingPatchVersion(semverVersion)
+
+	versionCve := ""
+
+	if isVersionALog4ShellVersion(semverVersion) {
+		if !strings.Contains(fileName, "JndiLookup.class") {
+			return
+		}
+		versionCve = constants.Log4ShellCve
+	}
+
+	if isVersionACVE202145046Version(semverVersion) {
+		if !strings.Contains(fileName, "JndiManager$JndiManagerFactory.class") {
+			return
+		}
+		versionCve = constants.CtxCve
+	}
+
+	if isVersionACVE201917571Version(semverVersion) {
+		if !strings.Contains(fileName, "SocketNode.class") {
+			return
+		}
+		versionCve = constants.Log4j1RceCve
+	}
+
+	if versionCve == "" {
+		return
+	}
+
+	fileHash, err := util.HexEncodedSha256FromReader(reader)
+	if err != nil {
+		log.Warn().
+			Str("fieName", fileName).
+			Str("path", filePath).
+			Err(err).
+			Msg("unable to hash file")
+		return
+	}
+
+	log.Log().
+		Str("path", filePath).
+		Str("fileName", fileName).
+		Str("fileHash", fileHash).
+		Msg("identified library version")
+
+	if versionCve == "" {
+		log.Debug().
+			Str("hash", fileHash).
+			Str("version", version).
+			Msg("Skipping version as it is not vulnerable to any known CVE")
+		return nil
+	}
+
+	finding = &types.Finding{
+		Path:        filePath,
+		FileName:    fileName,
+		Hash:        fileHash,
+		Version: semverVersion,
+		CVE: versionCve,
+	}
+	return
+}

tools/log4shell/commands/analyze.go

@@ -0,0 +1,40 @@
+// Copyright 2021 by LunaSec (owned by Refinery Labs, Inc)
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+package commands
+
+import (
+	"github.com/lunasec-io/lunasec/tools/log4shell/analyze"
+	"github.com/lunasec-io/lunasec/tools/log4shell/findings"
+	"github.com/lunasec-io/lunasec/tools/log4shell/scan"
+	"github.com/urfave/cli/v2"
+)
+
+func AnalyzeCommand(c *cli.Context) error {
+	enableGlobalFlags(c)
+
+	searchDirs := c.Args().Slice()
+
+	processArchiveFile := analyze.ProcessArchiveFile
+
+	scanner := scan.NewLog4jDirectoryScanner([]string{}, false, processArchiveFile)
+
+	scannerFindings := scanner.Scan(searchDirs)
+
+	output := c.String("output")
+	if output != "" {
+		return findings.SerializeToFile(output, scannerFindings)
+	}
+	return nil
+}

tools/log4shell/commands/flags.go

@@ -0,0 +1,54 @@
+// Copyright 2021 by LunaSec (owned by Refinery Labs, Inc)
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+package commands
+
+import (
+	"fmt"
+	"github.com/lunasec-io/lunasec/tools/log4shell/constants"
+	"github.com/lunasec-io/lunasec/tools/log4shell/util"
+	"github.com/rs/zerolog"
+	"github.com/rs/zerolog/log"
+	"github.com/urfave/cli/v2"
+	"os"
+)
+
+func enableGlobalFlags(c *cli.Context) {
+	verbose := c.Bool("verbose")
+	ignoreWarnings := c.Bool("ignore-warnings")
+	debug := c.Bool("debug")
+
+	if verbose || debug {
+		zerolog.SetGlobalLevel(zerolog.DebugLevel)
+	}
+	if ignoreWarnings {
+		zerolog.SetGlobalLevel(zerolog.ErrorLevel)
+	}
+
+	if debug {
+		// include file and line number when logging
+		log.Logger = log.With().Caller().Logger()
+	}
+
+	jsonFlag := c.Bool("json")
+	if !jsonFlag {
+		// pretty print output to the console if we are not interested in parsable output
+		consoleOutput := zerolog.ConsoleWriter{Out: os.Stderr}
+		consoleOutput.FormatFieldName = func(i interface{}) string {
+			return fmt.Sprintf("\n\t%s: ", util.Colorize(constants.ColorBlue, i))
+		}
+		log.Logger = log.Output(consoleOutput)
+
+	}
+}