Skip to content
This repository has been archived by the owner on Jun 30, 2023. It is now read-only.

readdir META-INF/versions/9: not implemented #12

Closed
mslinn opened this issue Dec 29, 2021 · 7 comments · Fixed by #16
Closed

readdir META-INF/versions/9: not implemented #12

mslinn opened this issue Dec 29, 2021 · 7 comments · Fixed by #16

Comments

@mslinn
Copy link

mslinn commented Dec 29, 2021 

$ ./log4jscanner $work
/work/cadenza/play28-template/null/Coursier/cache/v1/https/repo1.maven.org/maven2/org/apache/logging/log4j/log4j-core/2.11.2/log4j-core-2.11.2.jar
2021/12/29 14:41:53 log4jscanner.go:119: Error: scanning /work/cadenza/play28-template/null/Coursier/cache/v1/https/repo1.maven.org/maven2/org/seleniumhq/selenium/selenium-api/3.141.59/selenium-api-3.141.59.jar: scanning jar: failed to check JAR: readdir META-INF/versions/9: not implemented
2021/12/29 14:41:54 log4jscanner.go:119: Error: scanning /work/cadenza/play28-template/null/Coursier/cache/v1/https/repo1.maven.org/maven2/org/seleniumhq/selenium/selenium-remote-driver/3.141.59/selenium-remote-driver-3.141.59.jar: scanning jar: failed to check JAR: readdir META-INF/versions/9: not implemented
panic: runtime error: index out of range [-1]

goroutine 1 [running]:
archive/zip.split(...)
        /usr/lib/go-1.17/src/archive/zip/reader.go:800
archive/zip.fileEntryLess({0x59bb38, 0x11}, {0xc00001fb01, 0xc0005149e8})
        /usr/lib/go-1.17/src/archive/zip/reader.go:773 +0x206
archive/zip.(*Reader).initFileList.func1.1(0xc0005149e8, 0xc000519590)
        /usr/lib/go-1.17/src/archive/zip/reader.go:768 +0x4f
sort.doPivot_func({0xc0005149e8, 0xc000519590}, 0x0, 0x12)
        /usr/lib/go-1.17/src/sort/zfuncversion.go:83 +0x14c
sort.quickSort_func({0xc0005149e8, 0xc000519590}, 0xc00010e9c0, 0x0, 0xc000514a28)
        /usr/lib/go-1.17/src/sort/zfuncversion.go:143 +0x85
sort.Slice({0x4c9720, 0xc00010e9c0}, 0x20)
        /usr/lib/go-1.17/src/sort/slice.go:20 +0x9f
archive/zip.(*Reader).initFileList.func1()
        /usr/lib/go-1.17/src/archive/zip/reader.go:768 +0x42a
sync.(*Once).doSlow(0x40ca07, 0x508d60)
        /usr/lib/go-1.17/src/sync/once.go:68 +0xd2
sync.(*Once).Do(...)
        /usr/lib/go-1.17/src/sync/once.go:59
archive/zip.(*Reader).initFileList(0x508ce0)
        /usr/lib/go-1.17/src/archive/zip/reader.go:738 +0x45
archive/zip.(*Reader).Open(0x4cfe00, {0x4e3665, 0xc0002783e0})
        /usr/lib/go-1.17/src/archive/zip/reader.go:783 +0x2c
io/fs.Stat({0x508da0, 0xc0002783e0}, {0x4e3665, 0x1})
        /usr/lib/go-1.17/src/io/fs/stat.go:25 +0x9c
io/fs.WalkDir({0x508da0, 0xc0002783e0}, {0x4e3665, 0x1}, 0xc000514e38)
        /usr/lib/go-1.17/src/io/fs/walk.go:108 +0x3c
github.com/google/log4jscanner/jar.(*checker).checkJAR(0xc000519560, {0x508da0, 0xc0002783e0}, 0x0, 0xc000514ee8)
        /work/log4jscanner/jar/jar.go:126 +0xce
github.com/google/log4jscanner/jar.Parse({0x508c80, 0xc000230070})
        /work/log4jscanner/jar/jar.go:60 +0x9d
github.com/google/log4jscanner/jar.(*walker).visit(0xc000515e10, {0xc0005bd810, 0xc0005bd760}, {0x50aa08, 0xc0000b7cc0})
        /work/log4jscanner/jar/walker.go:157 +0x46e
github.com/google/log4jscanner/jar.(*Walker).Walk.func1({0xc0005bd810, 0xa2}, {0x50aa08, 0xc0000b7cc0}, {0x0, 0x0})
        /work/log4jscanner/jar/walker.go:80 +0x8d
io/fs.walkDir({0x508fa0, 0xc00009e480}, {0xc0005bd810, 0xa2}, {0x50aa08, 0xc0000b7cc0}, 0xc000515e00)
        /usr/lib/go-1.17/src/io/fs/walk.go:66 +0x83
io/fs.walkDir({0x508fa0, 0xc00009e480}, {0xc000026630, 0x86}, {0x50aa08, 0xc0000b7ac0}, 0xc000515e00)
        /usr/lib/go-1.17/src/io/fs/walk.go:85 +0x27f
io/fs.walkDir({0x508fa0, 0xc00009e480}, {0xc000028380, 0x80}, {0x50aa08, 0xc0000b7700}, 0xc000515e00)
        /usr/lib/go-1.17/src/io/fs/walk.go:85 +0x27f
io/fs.walkDir({0x508fa0, 0xc00009e480}, {0xc000028280, 0x76}, {0x50aa08, 0xc000274640}, 0xc000515e00)
        /usr/lib/go-1.17/src/io/fs/walk.go:85 +0x27f
io/fs.walkDir({0x508fa0, 0xc00009e480}, {0xc0002bc0e0, 0x6e}, {0x50aa08, 0xc0002b0740}, 0xc000515e00)
        /usr/lib/go-1.17/src/io/fs/walk.go:85 +0x27f
io/fs.walkDir({0x508fa0, 0xc00009e480}, {0xc0002bc310, 0x6a}, {0x50aa08, 0xc0002b0680}, 0xc000515e00)
        /usr/lib/go-1.17/src/io/fs/walk.go:85 +0x27f
io/fs.walkDir({0x508fa0, 0xc00009e480}, {0xc0002bc070, 0x63}, {0x50aa08, 0xc00031a600}, 0xc000515e00)
        /usr/lib/go-1.17/src/io/fs/walk.go:85 +0x27f
io/fs.walkDir({0x508fa0, 0xc00009e480}, {0xc00031ccc0, 0x53}, {0x50aa08, 0xc00031a5c0}, 0xc000515e00)
        /usr/lib/go-1.17/src/io/fs/walk.go:85 +0x27f
io/fs.walkDir({0x508fa0, 0xc00009e480}, {0xc00031f2c0, 0x4d}, {0x50aa08, 0xc00031a580}, 0xc000515e00)
        /usr/lib/go-1.17/src/io/fs/walk.go:85 +0x27f
io/fs.walkDir({0x508fa0, 0xc00009e480}, {0xc00031f220, 0x4a}, {0x50aa08, 0xc00031a540}, 0xc000515e00)
        /usr/lib/go-1.17/src/io/fs/walk.go:85 +0x27f
io/fs.walkDir({0x508fa0, 0xc00009e480}, {0xc00031f0e0, 0x44}, {0x50aa08, 0xc00031a500}, 0xc000515e00)
        /usr/lib/go-1.17/src/io/fs/walk.go:85 +0x27f
io/fs.walkDir({0x508fa0, 0xc00009e480}, {0xc00001f7c0, 0x3b}, {0x50aa08, 0xc0002754c0}, 0xc000515e00)
        /usr/lib/go-1.17/src/io/fs/walk.go:85 +0x27f
io/fs.walkDir({0x508fa0, 0xc00009e480}, {0xc000657a40, 0x36}, {0x50aa08, 0xc0002854c0}, 0xc000515e00)
        /usr/lib/go-1.17/src/io/fs/walk.go:85 +0x27f
io/fs.walkDir({0x508fa0, 0xc00009e480}, {0xc0000b01b0, 0x26}, {0x50aa08, 0xc00002d000}, 0xc000515e00)
        /usr/lib/go-1.17/src/io/fs/walk.go:85 +0x27f
io/fs.walkDir({0x508fa0, 0xc00009e480}, {0xc0004c4258, 0x17}, {0x50aa08, 0xc0000b6b00}, 0xc000515e00)
        /usr/lib/go-1.17/src/io/fs/walk.go:85 +0x27f
io/fs.walkDir({0x508fa0, 0xc00009e480}, {0xc00040e220, 0xb}, {0x50aa08, 0xc0000b6380}, 0xc000515e00)
        /usr/lib/go-1.17/src/io/fs/walk.go:85 +0x27f
io/fs.walkDir({0x508fa0, 0xc00009e480}, {0x4e3665, 0x1}, {0x50a9d0, 0xc00009e490}, 0xc000515e00)
        /usr/lib/go-1.17/src/io/fs/walk.go:85 +0x27f
io/fs.WalkDir({0x508fa0, 0xc00009e480}, {0x4e3665, 0x1}, 0xc00009ae00)
        /usr/lib/go-1.17/src/io/fs/walk.go:112 +0xde
github.com/google/log4jscanner/jar.(*Walker).Walk(0xc0000a02a0, {0x7ffdda5c1f73, 0xb})
        /work/log4jscanner/jar/walker.go:72 +0xba
main.main()
        /work/log4jscanner/log4jscanner.go:135 +0x61e
@ericchiang
Copy link
Member

No idea where the panic is coming from. Looks like it might actually be from the zip package?

panic: runtime error: index out of range [-1]

Can you attach the JAR that's causing the "not implemented" to this issue? If not, can you run zipinfo against it and grep for "META-INF/versions/9"

zipinfo /work/cadenza/play28-template/null/Coursier/cache/v1/https/repo1.maven.org/maven2/org/seleniumhq/selenium/selenium-api/3.141.59/selenium-api-3.141.59.jar | grep META_INF

@mslinn
Copy link
Author

mslinn commented Dec 29, 2021

I corrected the command you suggested (typo):

$ zipinfo /work/cadenza/play28-template/null/Coursier/cache/v1/https/repo1.maven.org/maven2/org/apache/logging/log4j/log4j-core/2.11.2/log4j-core-2.11.2.jar | grep META-INF/versions/9
drwxr-xr-x  2.0 unx        0 b- stor 19-Feb-05 18:12 META-INF/versions/9/
drwxr-xr-x  2.0 unx        0 b- stor 19-Feb-05 18:12 META-INF/versions/9/org/
drwxr-xr-x  2.0 unx        0 b- stor 19-Feb-05 18:12 META-INF/versions/9/org/apache/
drwxr-xr-x  2.0 unx        0 b- stor 19-Feb-05 18:12 META-INF/versions/9/org/apache/logging/
drwxr-xr-x  2.0 unx        0 b- stor 19-Feb-05 18:12 META-INF/versions/9/org/apache/logging/log4j/
drwxr-xr-x  2.0 unx        0 b- stor 19-Feb-05 18:12 META-INF/versions/9/org/apache/logging/log4j/core/
drwxr-xr-x  2.0 unx        0 b- stor 19-Feb-05 18:12 META-INF/versions/9/org/apache/logging/log4j/core/util/
-rw-r--r--  2.0 unx     1090 bl defN 19-Feb-05 18:11 META-INF/versions/9/org/apache/logging/log4j/core/util/SystemClock.class

@ericchiang
Copy link
Member

As crazy as this sounds, this might be a bug in archive/zip. We've hit those before.

log4jscanner/jar/jar.go

Lines 70 to 77 in edf4af1

// zipFS exists because of bugs hit in encoding/zip that causes reading "."
// to return "." as one of the entries. This in turn, makes fs.WalkDir()
// recurse infinitely.
//
// See: https://go.dev/issue/50179
type zipFS struct {
fs.FS
}

Panic is coming from

https://github.com/golang/go/blob/go1.17/src/archive/zip/reader.go#L800

Will probably need access to a reproducer to be able to debug further.

@mslinn
Copy link
Author

mslinn commented Dec 29, 2021

Maybe catch the error, report it, and continue?

@ericchiang ericchiang mentioned this issue Dec 29, 2021
Closed
@ericchiang
Copy link
Member

Opened golang/go#50390

@ericchiang ericchiang mentioned this issue Dec 29, 2021
Merged
@ericchiang ericchiang mentioned this issue Dec 30, 2021
Merged
@mslinn
Copy link
Author

mslinn commented Dec 30, 2021

The code was changed to look for returned error messages, and did not include a try/catch, probably because Go does not support that construct. Consequently, the program still can blow up, like this just now:

$ sudo /home/mslinn/.local/bin/log4jscanner /
/home/mslinn/.coursier/cache/v1/https/repo1.maven.org/maven2/org/apache/logging/log4j/log4j-core/2.11.2/log4j-core-2.11.2.jar
/home/mslinn/.ivy2/cache/org.apache.logging.log4j/log4j-core/jars/log4j-core-2.1.jar
/home/mslinn/.ivy2/cache/org.apache.logging.log4j/log4j-core/jars/log4j-core-2.11.2.jar
/home/mslinn/.ivy2/cache/org.apache.logging.log4j/log4j-core/jars/log4j-core-2.5.jar
/home/mslinn/.ivy2/cache/org.apache.logging.log4j/log4j-core/jars/log4j-core-2.8.1.jar
panic: runtime error: index out of range [-1]

goroutine 1 [running]:
archive/zip.split(...)
        /usr/lib/go-1.17/src/archive/zip/reader.go:800
archive/zip.fileEntryLess({0xc00019de50, 0x4d8aa0}, {0x5a0b58, 0xc00024a000})
        /usr/lib/go-1.17/src/archive/zip/reader.go:774 +0x1e5
archive/zip.(*Reader).initFileList.func1.1(0xc00056efa8, 0x6ed)
        /usr/lib/go-1.17/src/archive/zip/reader.go:768 +0x4f
sort.doPivot_func({0xc00056efa8, 0xc0000a0300}, 0x0, 0x6f9)
        /usr/lib/go-1.17/src/sort/zfuncversion.go:89 +0x222
sort.quickSort_func({0xc00056efa8, 0xc0000a0300}, 0xc00000c048, 0x0, 0xc00056efe8)
        /usr/lib/go-1.17/src/sort/zfuncversion.go:143 +0x85
sort.Slice({0x4cb7e0, 0xc00000c048}, 0x700)
        /usr/lib/go-1.17/src/sort/slice.go:20 +0x9f
archive/zip.(*Reader).initFileList.func1()
        /usr/lib/go-1.17/src/archive/zip/reader.go:768 +0x42a
sync.(*Once).doSlow(0x40ca07, 0x50bbe0)
        /usr/lib/go-1.17/src/sync/once.go:68 +0xd2
sync.(*Once).Do(...)
        /usr/lib/go-1.17/src/sync/once.go:59
archive/zip.(*Reader).initFileList(0x50bb60)
        /usr/lib/go-1.17/src/archive/zip/reader.go:738 +0x45
archive/zip.(*Reader).Open(0x4d1fe0, {0x4e5845, 0xc000a92000})
        /usr/lib/go-1.17/src/archive/zip/reader.go:783 +0x2c
io/fs.Stat({0x50bc20, 0xc000a92000}, {0x4e5845, 0x1})
        /usr/lib/go-1.17/src/io/fs/stat.go:25 +0x9c
io/fs.WalkDir({0x50bc20, 0xc000a92000}, {0x4e5845, 0x1}, 0xc00056f3f8)
        /usr/lib/go-1.17/src/io/fs/walk.go:108 +0x3c
github.com/google/log4jscanner/jar.(*checker).checkJAR(0xc0000a02d0, {0x50bc20, 0xc000a92000}, 0x0, 0xc00056f4a8)
        /mnt/_/work/log4jscanner/jar/jar.go:127 +0xce
github.com/google/log4jscanner/jar.Parse({0x50bb00, 0xc00028a150})
        /mnt/_/work/log4jscanner/jar/jar.go:61 +0x9d
github.com/google/log4jscanner/jar.(*walker).visit(0xc00056fe10, {0xc0000e2060, 0xc0000e2000}, {0x50d8e8, 0xc00035cb40})
        /mnt/_/work/log4jscanner/jar/walker.go:157 +0x46e
github.com/google/log4jscanner/jar.(*Walker).Walk.func1({0xc0000e2060, 0x57}, {0x50d8e8, 0xc00035cb40}, {0x0, 0x0})
        /mnt/_/work/log4jscanner/jar/walker.go:80 +0x8d
io/fs.walkDir({0x50be20, 0xc00009e480}, {0xc0000e2060, 0x57}, {0x50d8e8, 0xc00035cb40}, 0xc00056fe00)
        /usr/lib/go-1.17/src/io/fs/walk.go:66 +0x83
io/fs.walkDir({0x50be20, 0xc00009e480}, {0xc000783ec0, 0x35}, {0x50d8e8, 0xc000024840}, 0xc00056fe00)
        /usr/lib/go-1.17/src/io/fs/walk.go:85 +0x27f
io/fs.walkDir({0x50be20, 0xc00009e480}, {0xc000286180, 0x30}, {0x50d8e8, 0xc000024040}, 0xc00056fe00)
        /usr/lib/go-1.17/src/io/fs/walk.go:85 +0x27f
io/fs.walkDir({0x50be20, 0xc00009e480}, {0xc000236690, 0x21}, {0x50d8e8, 0xc00035d2c0}, 0xc00056fe00)
        /usr/lib/go-1.17/src/io/fs/walk.go:85 +0x27f
io/fs.walkDir({0x50be20, 0xc00009e480}, {0xc00001e1b0, 0x17}, {0x50d8e8, 0xc00034a340}, 0xc00056fe00)
        /usr/lib/go-1.17/src/io/fs/walk.go:85 +0x27f
io/fs.walkDir({0x50be20, 0xc00009e480}, {0xc00001e0c0, 0x11}, {0x50d8e8, 0xc000358a40}, 0xc00056fe00)
        /usr/lib/go-1.17/src/io/fs/walk.go:85 +0x27f
io/fs.walkDir({0x50be20, 0xc00009e480}, {0xc000352250, 0xb}, {0x50d8e8, 0xc00034be80}, 0xc00056fe00)
        /usr/lib/go-1.17/src/io/fs/walk.go:85 +0x27f
io/fs.walkDir({0x50be20, 0xc00009e480}, {0xc0003521f8, 0x4}, {0x50d8e8, 0xc0000b8780}, 0xc00056fe00)
        /usr/lib/go-1.17/src/io/fs/walk.go:85 +0x27f
io/fs.walkDir({0x50be20, 0xc00009e480}, {0x4e5845, 0x1}, {0x50d8b0, 0xc00009e490}, 0xc00056fe00)
        /usr/lib/go-1.17/src/io/fs/walk.go:85 +0x27f
io/fs.WalkDir({0x50be20, 0xc00009e480}, {0x4e5845, 0x1}, 0xc00009ae00)
        /usr/lib/go-1.17/src/io/fs/walk.go:112 +0xde
github.com/google/log4jscanner/jar.(*Walker).Walk(0xc0000a02a0, {0x7ffc1f85e512, 0x1})
        /mnt/_/work/log4jscanner/jar/walker.go:72 +0xba
main.main()
        /mnt/_/work/log4jscanner/log4jscanner.go:135 +0x61e

@ericchiang
Copy link
Member

The readdir error was fixed.

Go has a recover() mechanism, but I don't think there's anything we want do about the panic without a reproducer. Again if you have a reproducer, upstream would be very interested, since archive/zip isn't supposed to panic.

https://go.dev/blog/defer-panic-and-recover

With #20, we've switched away from using fs.WalkDir which may also help.

Sign up for free to subscribe to this conversation on GitHub. Already have an account? Sign in.
Assignees
No one assigned
Labels
None yet
Projects
None yet
Milestone
No milestone
Development

Successfully merging a pull request may close this issue.

jar: add workaround for malformed ZIP directory entries ericchiang/log4jscanner
2 participants
@mslinn @ericchiang