Skip to content

Commit

Permalink
tar: Allow more than 2 zero end blocks at end
Browse files Browse the repository at this point in the history
  • Loading branch information
@wader
wader committed Jan 18, 2022
1 parent 39d4d84 commit ba1edef
Show file tree
Hide file tree
Showing 4 changed files with 57 additions and 10 deletions.
22 changes: 16 additions & 6 deletions format/tar/tar.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,8 @@ func tarDecode(d *decode.D, in interface{}) interface{} {

// end marker is 512*2 zero bytes
endMarker := [blockBytes * 2]byte{}
endMarkerFound := false
var endMarkerStart int64
var endMarkerEnd int64
filesCount := 0

d.FieldArray("files", func(d *decode.D) {
Expand Down Expand Up @@ -87,15 +88,24 @@ func tarDecode(d *decode.D, in interface{}) interface{} {
})
filesCount++

bs := d.PeekBytes(blockBytes * 2)
if bytes.Equal(bs, endMarker[:]) {
endMarkerFound = true
if d.BitsLeft() >= int64(len(endMarker))*8 && bytes.Equal(d.PeekBytes(len(endMarker)), endMarker[:]) {
endMarkerStart = d.Pos()
// consensus seems to be to allow more than 2 zero blocks at end
d.SeekRel(int64(len(endMarker)) * 8)
zeroBlock := [blockBytes]byte{}
for d.BitsLeft() >= blockBytes*8 && bytes.Equal(d.PeekBytes(blockBytes), zeroBlock[:]) {
d.SeekRel(int64(len(zeroBlock)) * 8)
}
endMarkerEnd = d.Pos()
break
}
}
})
if endMarkerFound {
d.FieldRawLen("end_marker", int64(len(endMarker))*8)
endMarkerSize := endMarkerEnd - endMarkerStart
if endMarkerSize > 0 {
d.RangeFn(endMarkerStart, endMarkerSize, func(d *decode.D) {
d.FieldRawLen("end_marker", d.BitsLeft())
})
}

if filesCount == 0 {
Expand Down
39 changes: 39 additions & 0 deletions format/tar/testdata/no_end_marker.fqtest
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
# fq 'tobytes[0:.end_marker | tobytesrange.start]' test.tar > no_end_marker.tar
$ fq v no_end_marker.tar
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|.{}: no_end_marker.tar (tar) 0x0-0x3ff.7 (1024)
| | | files[0:1]: 0x0-0x3ff.7 (1024)
| | | [0]{}: file 0x0-0x3ff.7 (1024)
0x000|74 65 73 74 00 00 00 00 00 00 00 00 00 00 00 00|test............| name: "test" 0x0-0x63.7 (100)
* |until 0x63.7 (100) | |
0x060| 30 30 30 36 34 34 20 00 | 000644 . | mode: 420 ("000644 ") 0x64-0x6b.7 (8)
0x060| 30 30 30 37| 0007| uid: 501 ("000765 ") 0x6c-0x73.7 (8)
0x070|36 35 20 00 |65 . |
0x070| 30 30 30 30 32 34 20 00 | 000024 . | gid: 20 ("000024 ") 0x74-0x7b.7 (8)
0x070| 30 30 30 30| 0000| size: 6 ("00000000006 ") 0x7c-0x87.7 (12)
0x080|30 30 30 30 30 30 36 20 |0000006 |
0x080| 31 34 31 33 33 36 32 35| 14133625| mtime: 1634675538 ("14133625522 ") 0x88-0x93.7 (12)
0x090|35 32 32 20 |522 |
0x090| 30 31 32 32 32 34 00 20 | 012224. | chksum: 5268 ("012224") 0x94-0x9b.7 (8)
0x090| 30 | 0 | typeflag: "0" 0x9c-0x9c.7 (1)
0x090| 00 00 00| ...| linkname: "" 0x9d-0x100.7 (100)
0x0a0|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................|
* |until 0x100.7 (100) | |
0x100| 75 73 74 61 72 00 | ustar. | magic: "ustar" (valid) 0x101-0x106.7 (6)
0x100| 30 30 | 00 | version: 0 ("00") 0x107-0x108.7 (2)
0x100| 77 61 64 65 72 00 00| wader..| uname: "wader" 0x109-0x128.7 (32)
0x110|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................|
0x120|00 00 00 00 00 00 00 00 00 |......... |
0x120| 73 74 61 66 66 00 00| staff..| gname: "staff" 0x129-0x148.7 (32)
0x130|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................|
0x140|00 00 00 00 00 00 00 00 00 |......... |
0x140| 30 30 30 30 30 30 20| 000000 | devmajor: 0 ("000000 ") 0x149-0x150.7 (8)
0x150|00 |. |
0x150| 30 30 30 30 30 30 20 00 | 000000 . | devminor: 0 ("000000 ") 0x151-0x158.7 (8)
0x150| 00 00 00 00 00 00 00| .......| prefix: "" 0x159-0x1f3.7 (155)
0x160|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................|
* |until 0x1f3.7 (155) | |
0x1f0| 00 00 00 00 00 00 00 00 00 00 00 00| ............| header_block_padding: raw bits (all zero) 0x1f4-0x1ff.7 (12)
0x200|68 65 6c 6c 6f 0a |hello. | data: raw bits 0x200-0x205.7 (6)
0x200| 00 00 00 00 00 00 00 00 00 00| ..........| data_block_padding: raw bits (all zero) 0x206-0x3ff.7 (506)
0x210|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................|
* |until 0x3ff.7 (end) (506) | |
Binary file added format/tar/testdata/no_end_marker.tar
View file
Binary file not shown.
6 changes: 2 additions & 4 deletions format/tar/testdata/tar.fqtest
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,5 @@ $ fq -d tar v /test.tar
0x0200| 00 00 00 00 00 00 00 00 00 00| ..........| data_block_padding: raw bits (all zero) 0x206-0x3ff.7 (506)
0x0210|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................|
* |until 0x3ff.7 (506) | |
0x0400|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................| end_marker: raw bits 0x400-0x7ff.7 (1024)
* |until 0x7ff.7 (1024) | |
0x0800|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................| unknown0: raw bits 0x800-0x27ff.7 (8192)
* |until 0x27ff.7 (end) (8192) | |
0x0400|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................| end_marker: raw bits 0x400-0x27ff.7 (9216)
* |until 0x27ff.7 (end) (9216) | |

0 comments on commit ba1edef

Please sign in to comment.