diff --git a/pmtiles/loop.go b/pmtiles/loop.go index 2eea94f..e6775a6 100644 --- a/pmtiles/loop.go +++ b/pmtiles/loop.go @@ -31,7 +31,7 @@ const ( type Datum struct { bytes []byte - directory Directory + directory DirectoryV2 kind DatumKind hit bool } @@ -122,13 +122,13 @@ func (loop Loop) Start() { var size int ok := loop.fetcher.Do(key, func(reader io.Reader) { if req.kind == Root { - metadata, dir := ParseHeader(reader) + metadata, dir := ParseHeaderV2(reader) result = Datum{kind: Root, bytes: metadata, directory: dir} size = len(metadata) + dir.SizeBytes() } else if req.kind == Leaf { dir_bytes := make([]byte, key.rng.Length) io.ReadFull(reader, dir_bytes) - dir := ParseDirectory(dir_bytes) + dir := ParseDirectoryV2(dir_bytes) result = Datum{kind: Root, directory: dir} size = dir.SizeBytes() } else { diff --git a/pmtiles/reader.go b/pmtiles/readerv2.go similarity index 83% rename from pmtiles/reader.go rename to pmtiles/readerv2.go index 0129577..befcaef 100644 --- a/pmtiles/reader.go +++ b/pmtiles/readerv2.go @@ -17,13 +17,13 @@ type Range struct { Length uint32 } -type Directory struct { +type DirectoryV2 struct { Entries map[Zxy]Range LeafZ uint8 Leaves map[Zxy]Range } -func (d Directory) SizeBytes() int { +func (d DirectoryV2) SizeBytes() int { return 21*(len(d.Entries)+len(d.Leaves)) + 1 } @@ -42,7 +42,7 @@ func GetParentTile(tile Zxy, level uint8) Zxy { return Zxy{Z: level, X: uint32(x), Y: uint32(y)} } -func ParseEntry(b []byte) (uint8, Zxy, Range) { +func ParseEntryV2(b []byte) (uint8, Zxy, Range) { z_raw := b[0] x_raw := b[1:4] y_raw := b[4:7] @@ -60,11 +60,11 @@ func ParseEntry(b []byte) (uint8, Zxy, Range) { } } -func ParseDirectory(dir_bytes []byte) Directory { - the_dir := Directory{Entries: make(map[Zxy]Range), Leaves: make(map[Zxy]Range)} +func ParseDirectoryV2(dir_bytes []byte) DirectoryV2 { + the_dir := DirectoryV2{Entries: make(map[Zxy]Range), Leaves: make(map[Zxy]Range)} var maxz uint8 for i := 0; i < len(dir_bytes)/17; i++ { - leaf_z, zxy, rng := ParseEntry(dir_bytes[i*17 : i*17+17]) + leaf_z, zxy, rng := ParseEntryV2(dir_bytes[i*17 : i*17+17]) if leaf_z == 0 { the_dir.Entries[zxy] = rng } else { @@ -76,7 +76,7 @@ func ParseDirectory(dir_bytes []byte) Directory { return the_dir } -func ParseHeader(reader io.Reader) ([]byte, Directory) { +func ParseHeaderV2(reader io.Reader) ([]byte, DirectoryV2) { magic_num := make([]byte, 2) io.ReadFull(reader, magic_num) version := make([]byte, 2) @@ -91,6 +91,6 @@ func ParseHeader(reader io.Reader) ([]byte, Directory) { io.ReadFull(reader, metadata_bytes) dir_bytes := make([]byte, rootdir_len*17) io.ReadFull(reader, dir_bytes) - the_dir := ParseDirectory(dir_bytes) + the_dir := ParseDirectoryV2(dir_bytes) return metadata_bytes, the_dir } diff --git a/pmtiles/reader_test.go b/pmtiles/readerv2_test.go similarity index 100% rename from pmtiles/reader_test.go rename to pmtiles/readerv2_test.go diff --git a/pmtiles/subpyramid.go b/pmtiles/subpyramid.go index 8f2f840..55ea756 100644 --- a/pmtiles/subpyramid.go +++ b/pmtiles/subpyramid.go @@ -66,14 +66,14 @@ func SubpyramidXY(logger *log.Logger, input string, output string, z uint8, minX if err != nil { return } - metadata_bytes, root_directory := ParseHeader(f) + metadata_bytes, root_directory := ParseHeaderV2(f) var metadata Metadata json.Unmarshal(metadata_bytes, &metadata) metadata.Maxzoom = strconv.Itoa(int(z)) metadata.Bounds = bounds - writer := NewWriter(output) + // writer := NewWriter(output) if z >= root_directory.LeafZ { for key, rng := range root_directory.Leaves { @@ -87,7 +87,7 @@ func SubpyramidXY(logger *log.Logger, input string, output string, z uint8, minX io.ReadFull(f, dir_bytes) for i := 0; i < len(dir_bytes)/17; i++ { - leaf_z, lzxy, lrng := ParseEntry(dir_bytes[i*17 : i*17+17]) + leaf_z, lzxy, lrng := ParseEntryV2(dir_bytes[i*17 : i*17+17]) if leaf_z == 0 { if lzxy.Z <= z && Matches(z, minX, minY, maxX, maxY, lzxy) { _, err = f.Seek(int64(lrng.Offset), 0) @@ -96,7 +96,7 @@ func SubpyramidXY(logger *log.Logger, input string, output string, z uint8, minX } tile_data := make([]byte, lrng.Length) io.ReadFull(f, tile_data) - writer.WriteTile(lzxy, tile_data) + // writer.WriteTile(lzxy, tile_data) } } } @@ -111,10 +111,11 @@ func SubpyramidXY(logger *log.Logger, input string, output string, z uint8, minX } tile_data := make([]byte, rng.Length) io.ReadFull(f, tile_data) - writer.WriteTile(key, tile_data) + // writer.WriteTile(key, tile_data) } } new_metadata_bytes, _ := json.Marshal(metadata) - writer.Finalize(new_metadata_bytes) + _ = new_metadata_bytes + // writer.Finalize(new_metadata_bytes) } diff --git a/pmtiles/writer.go b/pmtiles/writer.go deleted file mode 100644 index c1bc7fa..0000000 --- a/pmtiles/writer.go +++ /dev/null @@ -1,120 +0,0 @@ -package pmtiles - -import ( - "encoding/binary" - "hash/fnv" - "os" - "sort" -) - -type Entry struct { - zxy Zxy - rng Range -} - -type Writer struct { - file *os.File - offset uint64 - tiles []Entry - hashToOffset map[uint64]uint64 -} - -type EntryAscending []Entry - -func (e EntryAscending) Len() int { - return len(e) -} - -func (e EntryAscending) Swap(i, j int) { - e[i], e[j] = e[j], e[i] -} - -func (e EntryAscending) Less(i, j int) bool { - if e[i].zxy.Z != e[j].zxy.Z { - return e[i].zxy.Z < e[j].zxy.Z - } - if e[i].zxy.X != e[j].zxy.X { - return e[i].zxy.X < e[j].zxy.X - } - return e[i].zxy.Y < e[j].zxy.Y -} - -func NewWriter(path string) Writer { - f, err := os.Create(path) - var empty []byte - var offset uint64 - offset = 512000 - empty = make([]byte, offset) - _, err = f.Write(empty) - if err != nil { - panic("Write failed") - } - hashToOffset := make(map[uint64]uint64) - return Writer{file: f, offset: offset, hashToOffset: hashToOffset} -} - -// return (uint32(binary.LittleEndian.Uint16(b[1:3])) << 8) + uint32(b[0]) - -func writeUint24(i uint32) []byte { - result := make([]byte, 3) - binary.LittleEndian.PutUint16(result[1:3], uint16(i>>8&0xFFFF)) - result[0] = uint8(i & 0xFF) - return result -} - -func writeUint48(i uint64) []byte { - result := make([]byte, 6) - binary.LittleEndian.PutUint32(result[2:6], uint32(i>>16&0xFFFFFFFF)) - binary.LittleEndian.PutUint16(result[0:2], uint16(i&0xFFFF)) - return result -} - -func (writer *Writer) writeEntry(entry Entry) { - binary.Write(writer.file, binary.LittleEndian, uint8(entry.zxy.Z)) - writer.file.Write(writeUint24(entry.zxy.X)) - writer.file.Write(writeUint24(entry.zxy.Y)) - writer.file.Write(writeUint48(entry.rng.Offset)) - binary.Write(writer.file, binary.LittleEndian, uint32(entry.rng.Length)) -} - -func (writer *Writer) WriteTile(zxy Zxy, data []byte) { - // TODO do gzip decompression here - hsh := fnv.New64a() - hsh.Write(data) - tileHash := hsh.Sum64() - - existingOffset, ok := writer.hashToOffset[tileHash] - - if ok { - writer.tiles = append(writer.tiles, Entry{zxy: zxy, rng: Range{Offset: existingOffset, Length: uint32(len(data))}}) - } else { - writer.file.Write(data) - writer.tiles = append(writer.tiles, Entry{zxy: zxy, rng: Range{Offset: writer.offset, Length: uint32(len(data))}}) - writer.hashToOffset[tileHash] = writer.offset - writer.offset += uint64(len(data)) - } -} - -func (writer *Writer) writeHeader(metadata []byte, numRootEntries int) { - _, _ = writer.file.Write([]byte{0x50, 0x4D}) // magic number - _ = binary.Write(writer.file, binary.LittleEndian, uint16(2)) // version - _ = binary.Write(writer.file, binary.LittleEndian, uint32(len(metadata))) // metadata length - _ = binary.Write(writer.file, binary.LittleEndian, uint16(numRootEntries)) // root dir entries - _, _ = writer.file.Write(metadata) -} - -func (writer *Writer) Finalize(metadata_bytes []byte) { - if len(writer.tiles) < 21845 { - _, _ = writer.file.Seek(0, 0) - writer.writeHeader(metadata_bytes, len(writer.tiles)) - - sort.Sort(EntryAscending(writer.tiles)) - - for _, entry := range writer.tiles { - writer.writeEntry(entry) - } - } else { - panic("Leaf directories not supported") - } - writer.file.Close() -} diff --git a/pmtiles/writer_test.go b/pmtiles/writer_test.go deleted file mode 100644 index fbd872f..0000000 --- a/pmtiles/writer_test.go +++ /dev/null @@ -1,34 +0,0 @@ -package pmtiles - -import ( - "bytes" - "testing" -) - -func TestWriteUint24(t *testing.T) { - var i uint32 - i = 16777215 - result := writeUint24(i) - if !bytes.Equal(result, []byte{255, 255, 255}) { - t.Errorf("result did not match, was %d", result) - } - i = 255 - result = writeUint24(i) - if !bytes.Equal(result, []byte{255, 0, 0}) { - t.Errorf("result did not match, was %d", result) - } -} - -func TestWriteUint48(t *testing.T) { - var i uint64 - i = 281474976710655 - result := writeUint48(i) - if !bytes.Equal(result, []byte{255, 255, 255, 255, 255, 255}) { - t.Errorf("result did not match, was %d", result) - } - i = 255 - result = writeUint48(i) - if !bytes.Equal(result, []byte{255, 0, 0, 0, 0, 0}) { - t.Errorf("result did not match, was %d", result) - } -}