Merge 5833327 into 73557e8

.travis.yml

@@ -0,0 +1,20 @@
+dist: trusty
+language: go
+go: '1.10'
+go_import_path: go.mozilla.org/mar
+before_install:
+- sudo apt-get -y install libnss3-tools
+- go get github.com/golang/lint/golint
+- go get golang.org/x/tools/cmd/cover
+- go get github.com/mattn/goveralls
+script:
+- make getkeys
+- make
+- goveralls -coverprofile=coverage.out -service=travis-ci -repotoken $COVERALLS_TOKEN
+- make getmarcorpus testmarcorpus
+# verify a mar signature with Firefox's signmar
+- |
+    crt="$(go run examples/sign.go /tmp/marworkdir/firefox-60.0esr-60.0.1esr.partial.mar /tmp/resigned.mar | grep 'rsa cert'|awk '{print $5}')"
+    mkdir /tmp/nssdb
+    certutil -d /tmp/nssdb -A -i "$crt" -n "testmar" -t ",,u"
+    LD_LIBRARY_PATH=tools/signmar/lib ./tools/signmar/signmar -d /tmp/nssdb -n testmar -v /tmp/resigned.mar

Makefile

@@ -0,0 +1,50 @@
+all: lint vet test getsamplemar testparser testsigner
+
+lint:
+	golint go.mozilla.org/mar
+
+vet:
+	go vet -composites=false go.mozilla.org/mar
+
+test:
+	go test -covermode=count -coverprofile=coverage.out go.mozilla.org/mar
+
+coverage: test
+	go tool cover -html=coverage.out
+
+getkeys:
+	bash get_firefox_keys.sh
+
+getsamplemar:
+	@if [ ! -e firefox-60.0esr-60.0.1esr.partial.mar ]; then \
+		wget http://download.cdn.mozilla.net/pub/firefox/releases/60.0.1esr/update/win64/en-US/firefox-60.0esr-60.0.1esr.partial.mar ;\
+	fi
+
+testparser:
+	go run -ldflags "-X go.mozilla.org/mar.debug=true" examples/parse.go firefox-60.0esr-60.0.1esr.partial.mar 2>&1 | grep 'signature: OK, valid signature from release1_sha384'
+
+testsigner:
+	go run -ldflags "-X go.mozilla.org/mar.debug=true" examples/sign.go firefox-60.0esr-60.0.1esr.partial.mar /tmp/resigned.mar
+	go run examples/parse.go /tmp/resigned.mar
+
+getmarcorpus:
+	@if [ ! -e /tmp/marworkdir ]; then mkdir /tmp/marworkdir; fi
+	@if [ ! -e /tmp/marworkdir/firefox-1.5rc2-1.5.partial.mar ]; then wget -P /tmp/marworkdir http://download.cdn.mozilla.net/pub/firefox/releases/1.5/update/win32/en-US/firefox-1.5rc2-1.5.partial.mar; fi
+	@if [ ! -e /tmp/marworkdir/firefox-10.0esr-10.0.1esr.partial.mar ]; then wget -P /tmp/marworkdir http://download.cdn.mozilla.net/pub/firefox/releases/10.0.1esr/update/linux-x86_64/fr/firefox-10.0esr-10.0.1esr.partial.mar; fi
+	@if [ ! -e /tmp/marworkdir/firefox-2.0.0.1.complete.mar ]; then wget -P /tmp/marworkdir http://download.cdn.mozilla.net/pub/firefox/releases/2.0.0.1/update/win32/en-US/firefox-2.0.0.1.complete.mar; fi
+	@if [ ! -e /tmp/marworkdir/firefox-2.0-2.0.0.1.partial.mar ]; then wget -P /tmp/marworkdir http://download.cdn.mozilla.net/pub/firefox/releases/2.0.0.1/update/mac/ru/firefox-2.0-2.0.0.1.partial.mar; fi
+	@if [ ! -e /tmp/marworkdir/firefox-3.5.13-3.5.14.partial.mar ]; then wget -P /tmp/marworkdir http://download.cdn.mozilla.net/pub/firefox/releases/3.5.14/update/win32/fy-NL/firefox-3.5.13-3.5.14.partial.mar; fi
+	@if [ ! -e /tmp/marworkdir/firefox-36.0b4-36.0b5.partial.mar ]; then wget -P /tmp/marworkdir http://download.cdn.mozilla.net/pub/firefox/releases/36.0b5/update/linux-i686/ga-IE/firefox-36.0b4-36.0b5.partial.mar; fi
+	@if [ ! -e /tmp/marworkdir/firefox-4.0rc1-4.0rc2.partial.mar ]; then wget -P /tmp/marworkdir http://download.cdn.mozilla.net/pub/firefox/releases/4.0rc2/update/win32/sv-SE/firefox-4.0rc1-4.0rc2.partial.mar; fi
+	@if [ ! -e /tmp/marworkdir/firefox-60.0esr-60.0.1esr.partial.mar ]; then wget -P /tmp/marworkdir http://download.cdn.mozilla.net/pub/firefox/releases/60.0.1esr/update/win64/en-US/firefox-60.0esr-60.0.1esr.partial.mar; fi
+
+testmarcorpus:
+	for f in $$(ls /tmp/marworkdir/firefox*.mar); do go run examples/parse.go "$$f"; done
+
+fuzz: getmarcorpus
+	go get -u github.com/dvyukov/go-fuzz/...
+	go-fuzz-build go.mozilla.org/mar
+	go-fuzz -bin=mar-fuzz.zip -workdir=/tmp/marworkdir
+
+.PHONY: all lint vet test getkeys getsamplemar testparser
+

README.md

@@ -0,0 +1,21 @@
+# MARGO: Mozilla ARchive library written in Go
+
+[![Build Status](https://travis-ci.org/mozilla-services/margo.svg?branch=master)](https://travis-ci.org/mozilla-services/margo)
+[![GoDoc](https://godoc.org/go.mozilla.org/mar?status.svg)](https://godoc.org/go.mozilla.org/mar) 
+[![Coverage Status](https://coveralls.io/repos/github/mozilla-services/margo/badge.svg?branch=master)](https://coveralls.io/github/mozilla-services/margo?branch=master)
+
+`import "go.mozilla.org/mar"`
+
+**Requires Go 1.10**
+
+Margo is a fairly secure MAR parser written to allow
+[autograph](https://github.com/mozilla-services/autograph) to sign Firefox
+MAR files. Its primary focus is signature, but it can also be used to parse,
+create and verify signatures on existing MAR files.
+
+Take a look at `example_test.go` for a taste of the API, or run the command line
+tools under `examples/`.
+
+## FAQ
+### Why is it called "margo"?
+it's subtle: it's a "mar" library, written in "go". get it? "margo"!

abuse_test.go

@@ -0,0 +1,91 @@
+package mar
+
+import (
+	"testing"
+)
+
+// firefox's signmar is happy to reference the same content block from
+// multiple index entries, which could be used as a zip bomb to create
+// a fraudulent mar file that decompresses to many times its original
+// size. Margo hash checks in place to refuse to unmarshal such files.
+// see also:
+//	BLRG-PT-18-013: DoS by Overly Large Files in MAR
+//	https://bugzilla.mozilla.org/show_bug.cgi?id=1468556
+func TestDosByLargeFile(t *testing.T) {
+	dosMar := File{
+		MarID: "MAR1",
+		Content: map[string]Entry{
+			"/foo/bar": Entry{
+				Data: []byte("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"),
+			},
+		},
+		Index: []IndexEntry{
+			IndexEntry{IndexEntryHeader{Flags: 0640}, "/foo/bar"},
+			IndexEntry{IndexEntryHeader{Flags: 0640}, "/foo/bar"},
+			IndexEntry{IndexEntryHeader{Flags: 0640}, "/foo/bar"},
+			IndexEntry{IndexEntryHeader{Flags: 0640}, "/foo/bar"},
+			IndexEntry{IndexEntryHeader{Flags: 0640}, "/foo/bar"},
+			IndexEntry{IndexEntryHeader{Flags: 0640}, "/foo/bar"},
+			IndexEntry{IndexEntryHeader{Flags: 0640}, "/foo/bar"},
+			IndexEntry{IndexEntryHeader{Flags: 0640}, "/foo/bar"},
+			IndexEntry{IndexEntryHeader{Flags: 0640}, "/foo/bar"},
+			IndexEntry{IndexEntryHeader{Flags: 0640}, "/foo/bar"},
+			IndexEntry{IndexEntryHeader{Flags: 0640}, "/foo/bar"},
+			IndexEntry{IndexEntryHeader{Flags: 0640}, "/foo/bar"},
+			IndexEntry{IndexEntryHeader{Flags: 0640}, "/foo/bar"},
+			IndexEntry{IndexEntryHeader{Flags: 0640}, "/foo/bar"},
+			IndexEntry{IndexEntryHeader{Flags: 0640}, "/foo/bar"},
+		},
+	}
+	o, err := dosMar.Marshal()
+	if err != nil {
+		t.Fatal(err)
+	}
+	var reparsed File
+	err = Unmarshal(o, &reparsed)
+	if err == nil {
+		t.Fatal("expected to fail with duplicate content read but succeeded", err)
+	}
+	if err != errCursorStartAlreadyRead {
+		t.Fatalf("expected to fail with duplicate content read but failed with: %v", err)
+	}
+}
+
+func TestBadIndexReference(t *testing.T) {
+	dosMar := File{
+		MarID: "MAR1",
+		Content: map[string]Entry{
+			"/foo/bar": Entry{
+				Data: []byte("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"),
+			},
+		},
+		Index: []IndexEntry{
+			IndexEntry{IndexEntryHeader{Flags: 0640}, "/does/not/exist"},
+		},
+	}
+	_, err := dosMar.Marshal()
+	if err == nil {
+		t.Fatalf("expected to fail with %q but succeeded", errIndexBadContentReference)
+	}
+	if err != errIndexBadContentReference {
+		t.Fatalf("expected to fail with %q but failed with %v", errIndexBadContentReference, err)
+	}
+}
+
+func TestEmptyIndex(t *testing.T) {
+	var f File
+	var emptyIndex = []byte("MAR1\x00\x00\x00\x88000000000000" +
+		"00000000000000000000" +
+		"00000000000000000000" +
+		"00000000000000000000" +
+		"00000000000000000000" +
+		"00000000000000000000" +
+		"00000000000000000000")
+	err := Unmarshal(emptyIndex, &f)
+	if err == nil {
+		t.Fatalf("expected to fail with %q but succeeded", errIndexTooSmall)
+	}
+	if err != errIndexTooSmall {
+		t.Fatalf("expected to fail with %q but failed with %v", errIndexTooSmall, err)
+	}
+}

doc.go

@@ -0,0 +1,47 @@
+/*
+Package mar implements support for the Mozilla ARchive format used by
+the Application Update Service of Firefox.
+
+The MAR format is specified at https://wiki.mozilla.org/Software_Update:MAR
+
+This package is primarily used to sign MARs by first parsing them via the
+Unmarshal function, then signing them with either RSA or ECDSA keys.
+
+	// read a MAR file from disk
+	input, _ := ioutil.ReadFile("/path/to/firefox.mar")
+	// parse it
+	_ = mar.Unmarshal(input, &file)
+	// prepare a signature using a given RSA key
+	file.PrepareSignature(rsaKey, rsaKey.Public())
+	// sign
+	_ = file.FinalizeSignatures()
+	// write out the signed mar file
+	output, _ := file.Marshal()
+	ioutil.WriteFile("/path/to/signed_firefox.mar", output, 0644)
+
+It can also be used to create new MARs and manipulate existing ones.
+
+	// create a new MAR
+	marFile := mar.New()
+	// Add data to the content section
+	marFile.AddContent([]byte("cariboumaurice"), "/foo/bar", 640)
+	// Add product information to the additional section
+	m.AddProductInfo("caribou maurice v1.2")
+	// Add random data to the additional section
+	m.AddAdditionalSection([]byte("foo bar baz"), uint32(1664))
+
+The MAR data structure exposes all internal fields, including offsets,
+sizes, etc. Those fields can be manipulated directly, but are ignored
+and recomputed when marshalling.
+
+The parser is fairly secure and will refuse to parse files that have
+duplicate content or try to reference the same data chunk multiple
+times. Doing so requires keeping track of previously parsed sections
+of a MAR, which induces a significant memory cost. Be mindful of allocated
+memory if you're going to parse a lot of very large MAR before the
+garbage collector has a chance to reclaim memory from previously
+parsed files.
+
+Various limits are enforced, take a look at errors.go for the details.
+*/
+package mar

errors.go

@@ -0,0 +1,60 @@
+package mar
+
+import (
+	"errors"
+	"fmt"
+)
+
+var (
+	limitMinFileSize uint64 = uint64(MarIDLen + OffsetToIndexLen + FileSizeLen + IndexHeaderLen + IndexEntryHeaderLen)
+
+	// the maximum size we'll agree to parse is 500MB.
+	// also set in Firefox at modules/libmar/src/mar_private.h#24-26
+	//
+	// People From The Future, if this isn't large enough for you, feel
+	// free to increase it, and have some self reflection because 640k
+	// oughta be enough for everybody!
+	limitMaxFileSize uint64 = 524288000
+
+	// filenames in the index shouldn't be longer than 1024 characters
+	limitFileNameLength = 1024
+
+	// an rsa signature on a 4096 bits key is 512 bytes, so by allowing 2k
+	// we could go up to 16k bit keys, which seems unlikely
+	// also set in Firefox at modules/libmar/src/mar_private.h#39-41
+	limitMaxSignatureSize uint32 = 2048
+
+	// additional data have a max size of 10MB
+	limitMaxAdditionalDataSize uint32 = 10485760
+)
+
+var (
+	errBadMarID                 = errors.New("mar ID must be MAR1")
+	errOffsetTooSmall           = errors.New("offset to index is too small to be valid")
+	errBadSigAlg                = errors.New("bad signature algorithm")
+	errInputTooShort            = errors.New("refusing to read more bytes than present in input")
+	errMalformedFileSize        = errors.New("the total file size does not match offset + index size")
+	errTooSmall                 = errors.New("the total file is below the minimum allowed of 32 bytes")
+	errTooBig                   = errors.New("the total file exceeds the maximum allowed of 500MB")
+	errSignatureTooBig          = errors.New("signature exceeds maximum allowed of 2048 bytes")
+	errSignatureUnknown         = errors.New("signature algorithm is unknown")
+	errAdditionalDataTooBig     = errors.New("additional data exceeds maximum allowed of 10MB")
+	errMalformedIndexFileName   = errors.New("malformed index is missing null terminator in file name")
+	errMalformedContentOverrun  = errors.New("malformed content offset and size overrun the end of the file")
+	errIndexFileNameTooBig      = errors.New("index file name exceeds the maximum length of 1024 characters")
+	errIndexFileNameOverrun     = errors.New("the length of the index file overruns the end of the file")
+	errIndexTooSmall            = errors.New("the index is smaller than the minimum allowed length of 12 bytes")
+	errIndexBadContentReference = errors.New("index entry references to content that does not exist")
+	errCursorStartAlreadyRead   = errors.New("start position has already been read in a previous chunk")
+	errCursorEndAlreadyRead     = errors.New("end position has already been read in a previous chunk")
+	errDupContent               = errors.New("a content entry with that name already exists")
+)
+
+// change that at runtime by setting -ldflags "-X go.mozilla.org/mar.debug=true"
+var debug = "false"
+
+func debugPrint(format string, a ...interface{}) {
+	if debug == "true" {
+		fmt.Printf(format, a...)
+	}
+}

errors_test.go

@@ -0,0 +1,9 @@
+package mar
+
+import "testing"
+
+func TestDebugPrint(t *testing.T) {
+	debug = "true"
+	debugPrint("debug is %s\n", debug)
+	debug = "false"
+}

example_test.go

@@ -0,0 +1,64 @@
+package mar_test
+
+import (
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/rsa"
+	"fmt"
+	"log"
+
+	"go.mozilla.org/mar"
+)
+
+func Example() {
+	marFile := mar.New()
+	marFile.AddContent([]byte("cariboumaurice"), "/foo/bar", 640)
+
+	// make a new rsa key and add it for signature
+	rsaPrivKey, err := rsa.GenerateKey(rand.Reader, 2048)
+	if err != nil {
+		log.Fatalf("rsa key generation failed with: %v", err)
+	}
+	marFile.PrepareSignature(rsaPrivKey, rsaPrivKey.Public())
+
+	// make a new ecdsa key and add it for signature
+	ecdsaPrivKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		log.Fatalf("ecdsa key generation failed with: %v", err)
+	}
+	marFile.PrepareSignature(ecdsaPrivKey, ecdsaPrivKey.Public())
+
+	// once both keys are added to the file, finalize the signature
+	err = marFile.FinalizeSignatures()
+	if err != nil {
+		log.Fatalf("mar signature failed with error: %v", err)
+	}
+
+	// write out the MAR file
+	outputMar, err := marFile.Marshal()
+	if err != nil {
+		log.Fatalf("mar marshalling failed with error: %v", err)
+	}
+
+	// reparse the MAR to make sure it goes through fine
+	var reparsedMar mar.File
+	err = mar.Unmarshal(outputMar, &reparsedMar)
+	if err != nil {
+		log.Fatalf("mar unmarshalling failed with error: %v", err)
+	}
+
+	// verify the signatures
+	err = reparsedMar.VerifySignature(rsaPrivKey.Public())
+	if err != nil {
+		log.Fatalf("failed to verify rsa signature: %v", err)
+	}
+	err = reparsedMar.VerifySignature(ecdsaPrivKey.Public())
+	if err != nil {
+		log.Fatalf("failed to verify ecdsa signature: %v", err)
+	}
+
+	fmt.Printf("MAR file signed and parsed without error")
+
+	// Output: MAR file signed and parsed without error
+}