Skip to content
Permalink
Browse files

Make mozlz4 public package of leatherman

  • Loading branch information...
frioux committed Apr 6, 2019
1 parent b5f5d26 commit 23af02f0f11b44e6ceec3cdbd2d25d0ac8dc1f41
Showing with 244 additions and 6 deletions.
  1. +1 −1 go.mod
  2. +0 −4 go.sum
  3. +1 −1 internal/tool/dumpmozlz4/dumpMozLz4.go
  4. +24 −0 pkg/mozlz4/examples_test.go
  5. +62 −0 pkg/mozlz4/mozlz4.go
  6. +156 −0 pkg/mozlz4/mozlz4_test.go
2 go.mod
@@ -5,7 +5,6 @@ require (
github.com/PuerkitoBio/goquery v1.5.0
github.com/erikdubbelboer/gspt v0.0.0-20190125194910-e68493906b83
github.com/frioux/mozcookiejar v0.0.2
github.com/frioux/mozlz4 v0.0.1
github.com/frioux/netrc v0.0.0-20190125054817-37b89b8d2a2d
github.com/frioux/shellquote v0.0.2
github.com/fsnotify/fsnotify v1.4.7
@@ -18,6 +17,7 @@ require (
github.com/mattn/go-sqlite3 v1.10.0
github.com/mmcdole/gofeed v1.0.0-beta2
github.com/mmcdole/goxpp v0.0.0-20181012175147-0068e33feabf // indirect
github.com/pierrec/lz4 v2.0.5+incompatible
github.com/pkg/errors v0.8.1
github.com/spf13/afero v1.2.0
github.com/stretchr/testify v1.3.0
4 go.sum
@@ -11,8 +11,6 @@ github.com/erikdubbelboer/gspt v0.0.0-20190125194910-e68493906b83 h1:ngHdSomn2My
github.com/erikdubbelboer/gspt v0.0.0-20190125194910-e68493906b83/go.mod h1:v6o7m/E9bfvm79dE1iFiF+3T7zLBnrjYjkWMa1J+Hv0=
github.com/frioux/mozcookiejar v0.0.2 h1:vAPbreMlaQsMPbedqv+h9OUMyLmtwWr9b1LqmOPcEYA=
github.com/frioux/mozcookiejar v0.0.2/go.mod h1:XA6YqSJViOtLsWH5dRSOj6nh/laBXu9EAdGFF6NEE9Q=
github.com/frioux/mozlz4 v0.0.1 h1:22TfRKdmz4QGmAxVB6IXKTMknPXjc1v+VdMaeqqTmPE=
github.com/frioux/mozlz4 v0.0.1/go.mod h1:nNYJvJfnYk9CR7BdhrVhccB7z8gYvUuOggNvqChYTk0=
github.com/frioux/netrc v0.0.0-20190125054817-37b89b8d2a2d h1:1529lO2sMK3+Pm7D6uRZcAhzp1WAqeyzXdHbg1RFncU=
github.com/frioux/netrc v0.0.0-20190125054817-37b89b8d2a2d/go.mod h1:Y4qsRBVQEhuujHdjar/vt5NUZ6tB31Ua5hx47En02+0=
github.com/frioux/shellquote v0.0.2 h1:CvQ1aMCS/xhhyGF4JIeA49bhE3W1s5XdBkwmhH3BceQ=
@@ -45,8 +43,6 @@ github.com/mmcdole/goxpp v0.0.0-20181012175147-0068e33feabf h1:sWGE2v+hO0Nd4yFU/
github.com/mmcdole/goxpp v0.0.0-20181012175147-0068e33feabf/go.mod h1:pasqhqstspkosTneA62Nc+2p9SOBBYAPbnmRRWPQ0V8=
github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I=
github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -5,7 +5,7 @@ import (
"io"
"os"

"github.com/frioux/mozlz4"
"github.com/frioux/leatherman/pkg/mozlz4"
"github.com/pkg/errors"
)

@@ -0,0 +1,24 @@
package mozlz4_test

import (
"fmt"
"io"
"os"

"github.com/frioux/leatherman/pkg/mozlz4"
)

func Example() {
file, err := os.Open(os.Args[1])
if err != nil {
fmt.Fprintf(os.Stderr, "Couldn't open: %s\n", err)
os.Exit(1)
}

r, err := mozlz4.NewReader(file)
_, err = io.Copy(os.Stdout, r)
if err != nil {
fmt.Fprintf(os.Stderr, "Couldn't copy: %s\n", err)
os.Exit(1)
}
}
@@ -0,0 +1,62 @@
package mozlz4

// Package mozlz4 implements the undocumented format used by Mozilla Firefox.

// The mozlz4 format (also known as jsonlz4 and json.lz4) is used by Firefox for
// various storage backends. The format is a magic header, a length, and an lz4
// compressed body.

import (
"bytes"
"encoding/binary"
"fmt"
"io"
"io/ioutil"

"github.com/pierrec/lz4"
"github.com/pkg/errors"
)

const magicHeader = "mozLz40\x00"

// Errors
var (
ErrWrongHeader = errors.New("no mozLz4 header")
ErrWrongSize = errors.New("header size incorrect")
)

// NewReader returns an io.Reader that decompresses the data from r.
func NewReader(r io.Reader) (io.Reader, error) {
header := make([]byte, len(magicHeader))
_, err := r.Read(header)
if err != nil {
return nil, errors.Wrap(err, "couldn't read header")
}
if string(header) != magicHeader {
return nil, ErrWrongHeader
}

var size uint32
err = binary.Read(r, binary.LittleEndian, &size)
if err != nil {
return nil, errors.Wrap(err, "couldn't read size")
}

src, err := ioutil.ReadAll(r)
if err != nil {
return nil, errors.Wrap(err, "couldn't read compressed data")
}

out := make([]byte, size)
sz, err := lz4.UncompressBlock(src, out)

if err != nil {
return nil, errors.Wrap(err, "couldn't decompress data")
}
// This could maybe be a warning or ignored entirely
if sz != int(size) {
return nil, errors.Wrap(ErrWrongSize, fmt.Sprintf("Header size %d, got %d", size, sz))
}

return bytes.NewReader(out), nil
}
@@ -0,0 +1,156 @@
package mozlz4

import (
"bytes"
"encoding/binary"
"io"
"io/ioutil"
"strings"
"testing"

"github.com/pierrec/lz4"
"github.com/pkg/errors"
"github.com/stretchr/testify/assert"
)

func errHasPrefix(t *testing.T, err error, prefix string) bool {
if !strings.HasPrefix(err.Error(), prefix) {
t.Logf("Error «%s» does not start with «%s»\n", err, prefix)
t.Fail()
return false
}
return true
}

func TestHappyPath(t *testing.T) {
str := "abcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyz"
r := strings.NewReader(str)
w := &bytes.Buffer{}

err := compress(r, w, len(str))
if err != nil {
t.Logf("Failed to compress data: %s\n", err)
t.Fail()
return
}

rt, err := NewReader(w)
if err != nil {
t.Logf("Failed to decompress data: %s\n", err)
t.Fail()
return
}

out, err := ioutil.ReadAll(rt)
if err != nil {
t.Logf("Failed to RedaAll data: %s\n", err)
t.Fail()
return
}

assert.Equal(t, str, string(out), "data roundtripped")
}

func TestWrongLength(t *testing.T) {
str := "abcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyzabcdefghijklmnopqrstuvzxyz"
r := strings.NewReader(str)
w := &bytes.Buffer{}

err := compress(r, w, 12+len(str))
if err != nil {
t.Logf("Failed to compress data: %s\n", err)
t.Fail()
return
}

_, err = NewReader(w)
assert.Equal(t, ErrWrongSize, errors.Cause(err))
}

func TestCantReadHeader(t *testing.T) {
r := bytes.NewReader(nil)
_, err := NewReader(r)
errHasPrefix(t, err, "couldn't read header")
}

func TestWrongHeader(t *testing.T) {
r := bytes.NewReader([]byte("lol"))
_, err := NewReader(r)
assert.Equal(t, ErrWrongHeader, errors.Cause(err))
}

func TestCantReadSize(t *testing.T) {
r := bytes.NewReader([]byte(magicHeader + "x"))
_, err := NewReader(r)
errHasPrefix(t, err, "couldn't read size")
}

func TestCantDecompress(t *testing.T) {
w := &bytes.Buffer{}
w.Write([]byte(magicHeader))
var size uint32 = 12
binary.Write(w, binary.LittleEndian, size)
w.Write([]byte{1, 2, 3, 4})
t.Log(w.Bytes())

r := bytes.NewReader(w.Bytes())
_, err := NewReader(r)

errHasPrefix(t, err, "couldn't decompress data")
}

func TestCantReadAll(t *testing.T) {
w := &bytes.Buffer{}
w.Write([]byte(magicHeader))
var size uint32 = 12
binary.Write(w, binary.LittleEndian, size)
w.Write([]byte{1, 2, 3, 4})

r := &ErrReader{Reader: bytes.NewReader(w.Bytes()), errAfter: 3}
_, err := NewReader(r)

errHasPrefix(t, err, "couldn't read compressed data")
}

type ErrReader struct {
io.Reader
errAfter int
}

func (r *ErrReader) Read(p []byte) (int, error) {
if r.errAfter == 0 {
return 0, errors.New("faked io error")
}
r.errAfter--
return r.Reader.Read(p)
}

func compress(src io.Reader, dst io.Writer, intendedSize int) error {
_, err := dst.Write([]byte(magicHeader))
if err != nil {
return errors.Wrap(err, "couldn't Write header")
}
b, err := ioutil.ReadAll(src)
if err != nil {
return errors.Wrap(err, "couldn't ReadAll to Compress")
}

err = binary.Write(dst, binary.LittleEndian, uint32(intendedSize))
if err != nil {
return errors.Wrap(err, "couldn't encode length")
}
dstBytes := make([]byte, 10*len(b))
sz, err := lz4.CompressBlockHC(b, dstBytes, -1)
if err != nil {
return errors.Wrap(err, "couldn't CompressBlock")
}
if sz == 0 {
return errors.New("data incompressible")
}
_, err = dst.Write(dstBytes[:sz])
if err != nil {
return errors.Wrap(err, "couldn't Write compressed data")
}

return nil
}

0 comments on commit 23af02f

Please sign in to comment.
You can’t perform that action at this time.