Feature/robust

binary.go

@@ -0,0 +1,90 @@
+package cpio
+
+import (
+	"encoding/binary"
+	"io"
+	"time"
+)
+
+const (
+	binaryHeaderLength = 26
+)
+
+func readInt16(b []byte) int {
+	return int(binary.LittleEndian.Uint16(b))
+}
+
+func int64FromInt32(b []byte) int64 {
+	// BigEndian order is called out in the cpio spec for the 16 bit words.
+	// This hard-codes LittleEndian Machine within the 16-bit words which
+	// should actually be parameterized by machine/archive
+	t := int64(binary.BigEndian.Uint32(
+		[]byte{
+			b[1],
+			b[0],
+			b[3],
+			b[2],
+		},
+	))
+	return t
+}
+
+func readBinaryHeader(r io.Reader) (*Header, error) {
+	// TODO: support binary-be
+
+	var buf [binaryHeaderLength - 2]byte // -2 for magic already read
+	if _, err := io.ReadFull(r, buf[:]); err != nil {
+		return nil, err
+	}
+	hdr := &Header{}
+
+	hdr.DeviceID = readInt16(buf[:2])
+	hdr.Inode = int64(readInt16(buf[2:4]))
+	hdr.Mode = FileMode(readInt16(buf[4:6]))
+	hdr.UID = readInt16(buf[6:8])
+	hdr.GID = readInt16(buf[8:10])
+	hdr.Links = readInt16(buf[10:12])
+	// skips rdev at buf[12:14]
+	hdr.ModTime = time.Unix(int64FromInt32(buf[14:18]), 0)
+	nameSize := readInt16(buf[18:20])
+	if nameSize < 1 {
+		return nil, ErrHeader
+	}
+
+	hdr.Size = int64(int64FromInt32(buf[20:])) // :24 is the end
+
+	name := make([]byte, nameSize)
+	if _, err := io.ReadFull(r, name); err != nil {
+		return nil, err
+	}
+	hdr.Name = string(name[:nameSize-1])
+	if hdr.Name == headerEOF {
+		return nil, io.EOF
+	}
+
+	// store padding between end of file and next header
+	hdr.pad = (2 - (hdr.Size % 2)) % 2
+
+	// skip to end of header/start of file
+	// +2 for magic bytes already read
+	pad := (2 - (2+len(buf)+len(name))%2) % 2
+	if pad > 0 {
+		if _, err := io.ReadFull(r, buf[:pad]); err != nil {
+			return nil, err
+		}
+	}
+
+	// read link name
+	if hdr.Mode&^ModePerm == ModeSymlink {
+		if hdr.Size < 1 {
+			return nil, ErrHeader
+		}
+		b := make([]byte, hdr.Size)
+		if _, err := io.ReadFull(r, b); err != nil {
+			return nil, err
+		}
+		hdr.Linkname = string(b)
+		hdr.Size = 0
+	}
+	return hdr, nil
+}

binary_test.go

@@ -0,0 +1,50 @@
+package cpio
+
+import (
+	"io"
+	"os"
+	"testing"
+)
+
+func TestReadBinary(t *testing.T) {
+	f, err := os.Open("testdata/test_binary.cpio")
+	if err != nil {
+		t.Fatalf("error opening test file: %v", err)
+	}
+	defer f.Close()
+
+	r := NewReader(f)
+	for {
+		_, err := r.Next()
+		if err == io.EOF {
+			return
+		}
+		if err != nil {
+			t.Errorf("error moving to the next header: %v", err)
+			return
+		}
+		// TODO: validate header fields
+	}
+}
+
+func TestReadBinaryRobust(t *testing.T) {
+	f, err := os.Open("testdata/test_binary_with_corruption.cpio")
+	if err != nil {
+		t.Fatalf("error opening test file: %v", err)
+	}
+	defer f.Close()
+
+	r := NewReader(f)
+	r.robust = true
+	for {
+		_, err := r.Next()
+		if err == io.EOF {
+			return
+		}
+		if err != nil {
+			t.Errorf("error moving to the next header: %v", err)
+			return
+		}
+		// TODO: validate header fields
+	}
+}

go.mod

@@ -0,0 +1,5 @@
+module github.com/jcrowgey/go-cpio
+
+go 1.12
+
+require github.com/cavaliercoder/go-cpio v0.0.0-20180626203310-925f9528c45e

go.sum

@@ -0,0 +1,2 @@
+github.com/cavaliercoder/go-cpio v0.0.0-20180626203310-925f9528c45e h1:hHg27A0RSSp2Om9lubZpiMgVbvn39bsUmW9U5h0twqc=
+github.com/cavaliercoder/go-cpio v0.0.0-20180626203310-925f9528c45e/go.mod h1:oDpT4efm8tSYHXV5tHSdRvBet/b/QzxZ+XyyPehvm3A=

header.go

@@ -31,6 +31,7 @@ const (
 
 var (
 	ErrHeader = errors.New("cpio: invalid cpio header")
+	ErrMagic  = errors.New("cpio: unrecognized magic number")
 )
 
 // A FileMode represents a file's mode and permission bits.

reader.go

@@ -1,18 +1,27 @@
 package cpio
 
 import (
+	"bytes"
 	"io"
 	"io/ioutil"
+	"log"
 )
 
+var magic map[string][]byte = map[string][]byte{
+	"binary-le": []byte{0xc7, 0x71},                         // 070707
+	"svr4":      []byte{0x30, 0x37, 0x30, 0x37, 0x30, 0x31}, // "070701"
+	"svr4-crc":  []byte{0x30, 0x37, 0x30, 0x37, 0x30, 0x32}, // "070702"
+}
+
 // A Reader provides sequential access to the contents of a CPIO archive. A CPIO
 // archive consists of a sequence of files. The Next method advances to the next
 // file in the archive (including the first), and then it can be treated as an
 // io.Reader to access the file's data.
 type Reader struct {
-	r   io.Reader // underlying file reader
-	hdr *Header   // current Header
-	eof int64     // bytes until the end of the current file
+	r      io.Reader // underlying file reader
+	hdr    *Header   // current Header
+	eof    int64     // bytes until the end of the current file
+	robust bool      // try to skip junk bytes when reading an archive
 }
 
 // NewReader creates a new Reader reading from r.
@@ -22,6 +31,15 @@ func NewReader(r io.Reader) *Reader {
 	}
 }
 
+// NewRobustReader creates a new Reader which will try to skip errors when
+// reading from r.
+func NewRobustReader(r io.Reader) *Reader {
+	return &Reader{
+		r:      r,
+		robust: true,
+	}
+}
+
 // Read reads from the current entry in the CPIO archive. It returns 0, io.EOF
 // when it reaches the end of that entry, until Next is called to advance to the
 // next entry.
@@ -56,7 +74,7 @@ func (r *Reader) Next() (*Header, error) {
 
 func (r *Reader) next() (*Header, error) {
 	r.eof = 0
-	hdr, err := readHeader(r.r)
+	hdr, err := readHeader(r.r, r.robust)
 	if err != nil {
 		return nil, err
 	}
@@ -66,7 +84,60 @@ func (r *Reader) next() (*Header, error) {
 }
 
 // ReadHeader creates a new Header, reading from r.
-func readHeader(r io.Reader) (*Header, error) {
-	// currently only SVR4 format is supported
-	return readSVR4Header(r)
+func readHeader(r io.Reader, robust bool) (*Header, error) {
+
+	var junk = 0
+	readOne := func() (*Header, error) {
+		// A binary magic is two bytes, an ascii magic is 6.
+		var binMagic [2]byte
+
+		if n, err := io.ReadFull(r, binMagic[:]); err != nil {
+			junk += n
+			return nil, err
+		}
+
+		if bytes.Equal(binMagic[:], magic["binary-le"]) {
+			// try to read a binary header
+			return readBinaryHeader(r)
+		}
+
+		// check if this starts an ascii magic
+		if bytes.Equal(binMagic[:], magic["svr4"][:2]) {
+			var ascMagic [6]byte
+			copy(ascMagic[:], binMagic[:])
+			if n, err := io.ReadFull(r, ascMagic[2:]); err != nil {
+				junk += 2 + n
+				return nil, err
+			}
+			if bytes.Equal(ascMagic[:], magic["svr4"]) {
+				return readSVR4Header(r, false)
+			} else if bytes.Equal(ascMagic[:], magic["svr4-crc"]) {
+				return readSVR4Header(r, true)
+			} else {
+				junk += 6
+				return nil, ErrMagic
+			}
+		}
+
+		junk += 2
+		return nil, ErrMagic
+	}
+
+	if robust {
+		for {
+			hdr, err := readOne()
+			if err == ErrMagic {
+				continue
+			}
+			if err != nil {
+				return nil, err
+			}
+			if junk > 0 {
+				log.Printf("skipped %d bytes of junk\n", junk)
+			}
+			return hdr, nil
+		}
+	} else {
+		return readOne()
+	}
 }

svr4.go

@@ -1,20 +1,18 @@
 package cpio
 
 import (
-	"bytes"
 	"fmt"
 	"io"
 	"strconv"
 	"time"
 )
 
 const (
-	svr4MaxNameSize = 4096 // MAX_PATH
-	svr4MaxFileSize = 4294967295
+	svr4MaxNameSize  = 4096 // MAX_PATH
+	svr4MaxFileSize  = 4294967295
+	svr4HeaderLength = 110
 )
 
-var svr4Magic = []byte{0x30, 0x37, 0x30, 0x37, 0x30, 0x31} // 070701
-
 func readHex(s string) int64 {
 	// errors are ignored and 0 returned
 	i, _ := strconv.ParseInt(s, 16, 64)
@@ -26,43 +24,32 @@ func writeHex(b []byte, i int64) {
 	copy(b, fmt.Sprintf("%08X", i))
 }
 
-func readSVR4Header(r io.Reader) (*Header, error) {
-	var buf [110]byte
-	if _, err := io.ReadFull(r, buf[:]); err != nil {
-		return nil, err
-	}
-
+func readSVR4Header(r io.Reader, hasCRC bool) (*Header, error) {
 	// TODO: check endianness
 
-	// check magic
-	hasCRC := false
-	if !bytes.HasPrefix(buf[:], svr4Magic[:5]) {
-		return nil, ErrHeader
-	}
-	if buf[5] == 0x32 { // '2'
-		hasCRC = true
-	} else if buf[5] != 0x31 { // '1'
-		return nil, ErrHeader
+	var buf [svr4HeaderLength - 6]byte // -6 for magic already read
+	if _, err := io.ReadFull(r, buf[:]); err != nil {
+		return nil, err
 	}
 
 	asc := string(buf[:])
 	hdr := &Header{}
 
-	hdr.Inode = readHex(asc[6:14])
-	hdr.Mode = FileMode(readHex(asc[14:22]))
-	hdr.UID = int(readHex(asc[22:30]))
-	hdr.GID = int(readHex(asc[30:38]))
-	hdr.Links = int(readHex(asc[38:46]))
-	hdr.ModTime = time.Unix(readHex(asc[46:54]), 0)
-	hdr.Size = readHex(asc[54:62])
+	hdr.Inode = readHex(asc[:8])
+	hdr.Mode = FileMode(readHex(asc[8:16]))
+	hdr.UID = int(readHex(asc[16:24]))
+	hdr.GID = int(readHex(asc[24:32]))
+	hdr.Links = int(readHex(asc[32:40]))
+	hdr.ModTime = time.Unix(readHex(asc[40:48]), 0)
+	hdr.Size = readHex(asc[48:56])
 	if hdr.Size > svr4MaxFileSize {
 		return nil, ErrHeader
 	}
-	nameSize := readHex(asc[94:102])
+	nameSize := readHex(asc[88:96])
 	if nameSize < 1 || nameSize > svr4MaxNameSize {
 		return nil, ErrHeader
 	}
-	hdr.Checksum = Checksum(readHex(asc[102:110]))
+	hdr.Checksum = Checksum(readHex(asc[96:104]))
 	if !hasCRC && hdr.Checksum != 0 {
 		return nil, ErrHeader
 	}
@@ -80,7 +67,8 @@ func readSVR4Header(r io.Reader) (*Header, error) {
 	hdr.pad = (4 - (hdr.Size % 4)) % 4
 
 	// skip to end of header/start of file
-	pad := (4 - (len(buf)+len(name))%4) % 4
+	// +6 for magic bytes already read
+	pad := (4 - (6+len(buf)+len(name))%4) % 4
 	if pad > 0 {
 		if _, err := io.ReadFull(r, buf[:pad]); err != nil {
 			return nil, err
@@ -104,15 +92,17 @@ func readSVR4Header(r io.Reader) (*Header, error) {
 }
 
 func writeSVR4Header(w io.Writer, hdr *Header) (pad int64, err error) {
-	var hdrBuf [110]byte
+	var hdrBuf [svr4HeaderLength]byte
 	for i := 0; i < len(hdrBuf); i++ {
 		hdrBuf[i] = '0'
 	}
-	magic := svr4Magic
+	var hMagic [6]byte
 	if hdr.Checksum != 0 {
-		magic[5] = 0x32
+		copy(hMagic[:], magic["svr4-crc"][:])
+	} else {
+		copy(hMagic[:], magic["svr4"][:])
 	}
-	copy(hdrBuf[:], magic)
+	copy(hdrBuf[:], hMagic[:])
 	writeHex(hdrBuf[6:14], hdr.Inode)
 	writeHex(hdrBuf[14:22], int64(hdr.Mode))
 	writeHex(hdrBuf[22:30], int64(hdr.UID))