Merge d107a16 into c8c24e7

.travis.yml

@@ -9,4 +9,5 @@ before_install:
 script:
 - make getkeys
 - make
+- make getmarcorpus testmarcorpus
 - goveralls -coverprofile=coverage.out -service=travis-ci -repotoken $COVERALLS_TOKEN

Makefile

@@ -27,5 +27,18 @@ testsigner:
 	go run -ldflags "-X go.mozilla.org/mar.debug=true" examples/sign.go firefox-60.0esr-60.0.1esr.partial.mar /tmp/resigned.mar
 	go run examples/parse.go /tmp/resigned.mar
 
+getmarcorpus:
+	wget -P /tmp http://download.cdn.mozilla.net/pub/firefox/releases/1.5/update/win32/en-US/firefox-1.5rc2-1.5.partial.mar
+	wget -P /tmp http://download.cdn.mozilla.net/pub/firefox/releases/10.0.1esr/update/linux-x86_64/fr/firefox-10.0esr-10.0.1esr.partial.mar
+	wget -P /tmp http://download.cdn.mozilla.net/pub/firefox/releases/2.0.0.1/update/win32/en-US/firefox-2.0.0.1.complete.mar
+	wget -P /tmp http://download.cdn.mozilla.net/pub/firefox/releases/2.0.0.1/update/mac/ru/firefox-2.0-2.0.0.1.partial.mar
+	wget -P /tmp http://download.cdn.mozilla.net/pub/firefox/releases/3.5.14/update/win32/fy-NL/firefox-3.5.13-3.5.14.partial.mar
+	wget -P /tmp http://download.cdn.mozilla.net/pub/firefox/releases/36.0b5/update/linux-i686/ga-IE/firefox-36.0b4-36.0b5.partial.mar
+	wget -P /tmp http://download.cdn.mozilla.net/pub/firefox/releases/4.0rc2/update/win32/sv-SE/firefox-4.0rc1-4.0rc2.partial.mar
+	wget -P /tmp http://download.cdn.mozilla.net/pub/firefox/releases/60.0.1esr/update/win64/en-US/firefox-60.0esr-60.0.1esr.partial.mar
+
+testmarcorpus:
+	for f in $$(ls /tmp/firefox*.mar); do go run examples/parse.go "$$f"; done
+
 .PHONY: all lint vet test getkeys getsamplemar testparser
 

errors.go

@@ -6,19 +6,22 @@ import (
 )
 
 var (
-	limitMinFileSize = MarIDLen + OffsetToIndexLen + SignaturesHeaderLen + AdditionalSectionsHeaderLen + IndexHeaderLen
+	limitMinFileSize uint64 = uint64(MarIDLen + OffsetToIndexLen + FileSizeLen + IndexHeaderLen + IndexEntryHeaderLen)
 
-	// the maximum size we'll agree to parse is 2GB.
+	// the maximum size we'll agree to parse is 500MB.
+	// also set in Firefox at modules/libmar/src/mar_private.h#24-26
+	//
 	// People From The Future, if this isn't large enough for you, feel
 	// free to increase it, and have some self reflection because 640k
 	// oughta be enough for everybody!
-	limitMaxFileSize uint64 = 2147483648
+	limitMaxFileSize uint64 = 524288000
 
 	// filenames in the index shouldn't be longer than 1024 characters
 	limitFileNameLength = 1024
 
 	// an rsa signature on a 4096 bits key is 512 bytes, so by allowing 2k
 	// we could go up to 16k bit keys, which seems unlikely
+	// also set in Firefox at modules/libmar/src/mar_private.h#39-41
 	limitMaxSignatureSize uint32 = 2048
 
 	// additional data have a max size of 10MB
@@ -31,14 +34,16 @@ var (
 	errBadSigAlg                = errors.New("bad signature algorithm")
 	errInputTooShort            = errors.New("refusing to read more bytes than present in input")
 	errMalformedFileSize        = errors.New("the total file size does not match offset + index size")
-	errTooBig                   = errors.New("the total file exceeds the maximum allowed of 2GB")
+	errTooSmall                 = errors.New("the total file is below the minimum allowed of 32 bytes")
+	errTooBig                   = errors.New("the total file exceeds the maximum allowed of 500MB")
 	errSignatureTooBig          = errors.New("signature exceeds maximum allowed of 2048 bytes")
 	errSignatureUnknown         = errors.New("signature algorithm is unknown")
 	errAdditionalDataTooBig     = errors.New("additional data exceeds maximum allowed of 10MB")
 	errMalformedIndexFileName   = errors.New("malformed index is missing null terminator in file name")
 	errMalformedContentOverrun  = errors.New("malformed content offset and size overrun the end of the file")
 	errIndexFileNameTooBig      = errors.New("index file name exceeds the maximum length of 1024 characters")
 	errIndexFileNameOverrun     = errors.New("the length of the index file overruns the end of the file")
+	errIndexTooSmall            = errors.New("the index is smaller than the minimum allowed length of 12 bytes")
 	errIndexBadContentReference = errors.New("index entry references to content that does not exist")
 	errCursorStartAlreadyRead   = errors.New("start position has already been read in a previous chunk")
 	errCursorEndAlreadyRead     = errors.New("end position has already been read in a previous chunk")

examples/parse.go

@@ -1,7 +1,6 @@
 package main
 
 import (
-	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"log"
@@ -21,11 +20,13 @@ func main() {
 	if err != nil {
 		log.Fatal(err)
 	}
-	out, err := json.MarshalIndent(file, "", "    ")
-	if err != nil {
-		log.Fatal(err)
+	fmt.Printf("%s\tsize=%d bytes\tsignatures=%d\tcontent=%d entries\tproduct=%q\trevision=%d\n",
+		file.MarID, file.Size,
+		file.SignaturesHeader.NumSignatures, len(file.Index),
+		file.ProductInformation, file.Revision)
+	if file.Revision < 2012 {
+		os.Exit(0)
 	}
-	fmt.Printf("%s\n", out)
 	validKeys, isSigned, err := file.VerifyWithFirefoxKeys()
 	if err != nil {
 		log.Fatal(err)

mar.go

@@ -18,10 +18,12 @@ const (
 	// to the beginning of the file
 	OffsetToIndexLen = 4
 
-	// SignaturesHeaderLen is the length of the signatures header
-	// The signature header contains the total size of the MAR file on 8 bytes
-	// and the number of signatures in the file on 4 bytes
-	SignaturesHeaderLen = 12
+	// FileSizeLen is a uint64 that contains the total size of the MAR in bytes
+	FileSizeLen = 8
+
+	// SignaturesHeaderLen is the length of the signatures header that
+	// contains the number of signatures in the MAR
+	SignaturesHeaderLen = 4
 
 	// SignatureEntryHeaderLen is the length of the header of each signature entry
 	// Each signature entry contains an algorithm and a size, each on 4 bytes
@@ -54,6 +56,7 @@ const (
 type File struct {
 	MarID                    string                   `json:"mar_id" yaml:"mar_id"`
 	OffsetToIndex            uint32                   `json:"offset_to_index" yaml:"offset_to_index"`
+	Size                     uint64                   `json:"size" yaml:"size"`
 	ProductInformation       string                   `json:"product_information,omitempty" yaml:"product_information,omitempty"`
 	SignaturesHeader         SignaturesHeader         `json:"signature_header" yaml:"signature_header"`
 	Signatures               []Signature              `json:"signatures" yaml:"signatures"`
@@ -62,16 +65,15 @@ type File struct {
 	IndexHeader              IndexHeader              `json:"index_header" yaml:"index_header"`
 	Index                    []IndexEntry             `json:"index" yaml:"index"`
 	Content                  map[string]Entry         `json:"-" yaml:"-"`
+	Revision                 int                      `json:"revision" yaml:"revision"`
 
 	// marshalForSignature is used to tell the marshaller to exclude
 	// signature data when preparing a file for signing
 	marshalForSignature bool
 }
 
-// SignaturesHeader contains the total file size and number of signatures in the MAR file
+// SignaturesHeader contains the number of signatures in the MAR file
 type SignaturesHeader struct {
-	// FileSize is the total size of the MAR file in bytes
-	FileSize uint64 `json:"file_size" yaml:"file_size"`
 	// NumSignatures is the count of signatures
 	NumSignatures uint32 `json:"num_signatures" yaml:"num_signatures"`
 }
@@ -159,18 +161,50 @@ type IndexEntryHeader struct {
 // New returns an initialized MAR data structure
 func New() *File {
 	return &File{
-		MarID:   "MAR1",
-		Content: make(map[string]Entry),
+		MarID:    "MAR1",
+		Content:  make(map[string]Entry),
+		Revision: 2012,
 	}
 }
 
-// Unmarshal takes an unparsed MAR file as input and parses it into a File struct
+// Unmarshal takes an unparsed MAR file as input and parses it into a File struct.
+// The MAR format is described at https://wiki.mozilla.org/Software_Update:MAR
+// but don't believe everything it says, because the format has changed over the
+// years to support more fields, and of course the MarID has not changed since.
+// There's a bit of magic in this function to detect which version of a MAR we're
+// dealing with, and store that in the Revision field of the file. 2005 is an old
+// MAR, 2012 is a current one with signatures and additional sections.
 func Unmarshal(input []byte, file *File) error {
-	if len(input) < limitMinFileSize {
-		return fmt.Errorf("input is smaller than minimum MAR size and cannot be parsed")
+	switch file.Size = uint64(len(input)); {
+	case file.Size < limitMinFileSize:
+		debugPrint("input=%d < limit=%d\n", file.Size, limitMinFileSize)
+		return errTooSmall
+	case file.Size > limitMaxFileSize:
+		debugPrint("input=%d > limit=%d\n", file.Size, limitMaxFileSize)
+		return errTooBig
 	}
+
 	p := newParser(input)
 
+	//  A modern MAR is composed of the following fields, in bytes:
+	//  0                   1
+	//  0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5
+	// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+...............
+	// | MAR ID| Offset|Total FileSize |Signatures|Add.Section|Content|Index
+	// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+...............
+	//
+	// except if we're dealing with an old MAR, in which case it's
+	//  0                   1
+	//  0 1 2 3 4 5 6 7 8 9 0 1 2 3 4
+	// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+.................
+	// | MAR ID| Offset|...Content...|IdxSize|[Idx Entries]
+	// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+.................
+	//
+	// We need to detect which type of MAR we're dealing with, so first
+	// we parse the MarID and Offset, then we jump to the first index
+	// entry to see if its offset to content starts right after the header,
+	// in which case we're dealing with an old MAR, otherwise it's a new one.
+
 	// Parse the MAR ID
 	marid := make([]byte, MarIDLen, MarIDLen)
 	err := p.parse(&marid, MarIDLen)
@@ -188,33 +222,90 @@ func Unmarshal(input []byte, file *File) error {
 		return fmt.Errorf("offset parsing failed: %v", err)
 	}
 
-	// Parse the Signature header
-	err = p.parse(&file.SignaturesHeader, SignaturesHeaderLen)
-	if err != nil {
-		return fmt.Errorf("signature header parsing failed: %v", err)
-	}
-
-	// Jump to the index header to parse it.
-	// We do that do get the index size and check that the total file size
-	// matches the filesize in the signature header
-	// This provides an extra sanity check to make sure we're not about to
-	// parse a malform MAR file with a size large enough to boil the oceans
-	// (because filesize is a uint64, for some reason...)
-	savedCursor := p.cursor
+	// parse the index
 	p.cursor = uint64(file.OffsetToIndex)
 	err = p.parse(&file.IndexHeader, IndexHeaderLen)
 	if err != nil {
 		return fmt.Errorf("index header parsing failed: %v", err)
 	}
-	if file.SignaturesHeader.FileSize != uint64(file.OffsetToIndex+file.IndexHeader.Size+IndexHeaderLen) {
-		return errMalformedFileSize
+	if file.IndexHeader.Size < IndexEntryHeaderLen {
+		return errIndexTooSmall
 	}
-	if file.SignaturesHeader.FileSize > limitMaxFileSize {
-		return errTooBig
+
+	for i := 0; ; i++ {
+		var (
+			idxEntryHeader IndexEntryHeader
+			idxEntry       IndexEntry
+		)
+		// don't read beyond the end of the file
+		if uint64(p.cursor) >= file.Size {
+			break
+		}
+		err = p.parse(&idxEntryHeader, IndexEntryHeaderLen)
+		if err != nil {
+			return fmt.Errorf("index entry parsing failed: %v", err)
+		}
+
+		idxEntry.Size = idxEntryHeader.Size
+		idxEntry.Flags = idxEntryHeader.Flags
+		idxEntry.OffsetToContent = idxEntryHeader.OffsetToContent
+		if uint64(idxEntry.OffsetToContent+idxEntry.Size) > file.Size {
+			return errMalformedContentOverrun
+		}
+
+		endNamePos := bytes.Index(input[p.cursor:], []byte("\x00"))
+
+		// apply some sanity checking on filenames.
+		// they shouldn't be longer than 1024 characters, and their length
+		// can't overrun the size of the input
+		if endNamePos < 0 {
+			return errMalformedIndexFileName
+		}
+		if endNamePos > limitFileNameLength {
+			return errIndexFileNameTooBig
+		}
+		if (p.cursor + uint64(endNamePos)) > file.Size {
+			return errIndexFileNameOverrun
+
+		}
+		idxEntry.FileName = string(input[p.cursor : p.cursor+uint64(endNamePos)])
+
+		// manually move the cursor to the end of the filename
+		p.cursor = p.cursor + uint64(endNamePos) + 1
+
+		file.Index = append(file.Index, idxEntry)
+	}
+
+	// evaluate the first index entry and if the offset to content is set to byte 8,
+	// we have an old MAR that has no signature or additional sections
+	if file.Index[0].OffsetToContent == MarIDLen+OffsetToIndexLen {
+		file.Revision = 2005
+		// use the input len as a file size since we don't have one in the headers
+		file.Size = uint64(len(input))
+		// skip the signature and additonal section parsing, we have none
+		goto parseContent
 	}
 
 	// go back to the beginning of the signatures block
-	p.cursor = savedCursor
+	p.cursor = MarIDLen + OffsetToIndexLen
+	file.Revision = 2012
+
+	// Parse the total file size header
+	err = p.parse(&file.Size, FileSizeLen)
+	if err != nil {
+		return fmt.Errorf("total file size header parsing failed: %v", err)
+	}
+	// make sure the file size is consistent with the offsets and index len
+	if file.Size != uint64(file.OffsetToIndex+file.IndexHeader.Size+IndexHeaderLen) {
+		debugPrint("filesize=%d; offset to index=%d; index size=%d\n",
+			file.Size, file.OffsetToIndex, file.IndexHeader.Size)
+		return errMalformedFileSize
+	}
+	// Parse the signatures header
+	err = p.parse(&file.SignaturesHeader, SignaturesHeaderLen)
+	if err != nil {
+		return fmt.Errorf("total file size header parsing failed: %v", err)
+	}
 
 	// Parse each signature and append them to the File
 	for i := uint32(0); i < file.SignaturesHeader.NumSignatures; i++ {
@@ -286,53 +377,8 @@ func Unmarshal(input []byte, file *File) error {
 		file.AdditionalSections = append(file.AdditionalSections, as)
 	}
 
-	// parse the index
-	p.cursor = uint64(file.OffsetToIndex + IndexHeaderLen)
-	for i := 0; ; i++ {
-		var (
-			idxEntryHeader IndexEntryHeader
-			idxEntry       IndexEntry
-		)
-		// don't read beyond the end of the file
-		if uint64(p.cursor) >= file.SignaturesHeader.FileSize {
-			break
-		}
-		err = p.parse(&idxEntryHeader, IndexEntryHeaderLen)
-		if err != nil {
-			return fmt.Errorf("index entry parsing failed: %v", err)
-		}
-
-		idxEntry.Size = idxEntryHeader.Size
-		idxEntry.Flags = idxEntryHeader.Flags
-		idxEntry.OffsetToContent = idxEntryHeader.OffsetToContent
-		if uint64(idxEntry.OffsetToContent+idxEntry.Size) > file.SignaturesHeader.FileSize {
-			return errMalformedContentOverrun
-		}
-
-		endNamePos := bytes.Index(input[p.cursor:], []byte("\x00"))
-
-		// apply some sanity checking on filenames.
-		// they shouldn't be longer than 1024 characters, and their length
-		// can't overrun the size of the input
-		if endNamePos < 0 {
-			return errMalformedIndexFileName
-		}
-		if endNamePos > limitFileNameLength {
-			return errIndexFileNameTooBig
-		}
-		if (p.cursor + uint64(endNamePos)) > file.SignaturesHeader.FileSize {
-			return errIndexFileNameOverrun
-
-		}
-		idxEntry.FileName = string(input[p.cursor : p.cursor+uint64(endNamePos)])
-
-		// manually move the cursor to the end of the filename
-		p.cursor = p.cursor + uint64(endNamePos) + 1
-
-		file.Index = append(file.Index, idxEntry)
-	}
-
 	// parse the content
+parseContent:
 	file.Content = make(map[string]Entry)
 	for _, idxEntry := range file.Index {
 		var entry Entry
@@ -379,17 +425,20 @@ func (file *File) Marshal() ([]byte, error) {
 	if err != nil {
 		return nil, err
 	}
-
 	err = binary.Write(buf, binary.BigEndian, file.OffsetToIndex)
 	if err != nil {
 		return nil, err
 	}
+	err = binary.Write(buf, binary.BigEndian, file.Size)
+	if err != nil {
+		return nil, err
+	}
 	err = binary.Write(buf, binary.BigEndian, file.SignaturesHeader)
 	if err != nil {
 		return nil, err
 	}
-	// start the cursor after the first 3 headers
-	offsetToContent = MarIDLen + OffsetToIndexLen + SignaturesHeaderLen
+	// start the cursor after the headers
+	offsetToContent = MarIDLen + OffsetToIndexLen + FileSizeLen + SignaturesHeaderLen
 
 	// Write the signatures
 	for _, sig := range file.Signatures {
@@ -496,9 +545,9 @@ func (file *File) Marshal() ([]byte, error) {
 	// this is basically the size of both the main and index buffer, but also the
 	// size of any future signatures if we're marshalling for signature (otherwise
 	// sigSizes is zero because the signature data is already in buf)
-	file.SignaturesHeader.FileSize = uint64(buf.Len() + finalIdxBuf.Len() + sigSizes)
+	file.Size = uint64(buf.Len() + finalIdxBuf.Len() + sigSizes)
 	fsizeBuf := new(bytes.Buffer)
-	err = binary.Write(fsizeBuf, binary.BigEndian, file.SignaturesHeader.FileSize)
+	err = binary.Write(fsizeBuf, binary.BigEndian, file.Size)
 	if err != nil {
 		return nil, err
 	}