Skip to content

Commit

Permalink
Merge pull request #18 from sfomuseum/return-error
Browse files Browse the repository at this point in the history 
Return errors instead of triggering fatal errors in library code
  • Loading branch information
@bdon
bdon committed Oct 13, 2022
2 parents 84e7b97 + 490d168 commit 0493afb
Show file tree
Hide file tree
Showing 9 changed files with 147 additions and 109 deletions.
27 changes: 23 additions & 4 deletions main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,11 @@ func main() {

switch os.Args[1] {
case "show":
pmtiles.Show(logger, os.Args[2:])
err := pmtiles.Show(logger, os.Args[2:])

if err != nil {
logger.Fatalf("Failed to show database, %v", err)
}
case "serve":
serveCmd := flag.NewFlagSet("serve", flag.ExitOnError)
port := serveCmd.String("p", "8080", "port to serve on")
Expand All @@ -37,7 +41,12 @@ func main() {
logger.Println("USAGE: serve [-p PORT] [-cors VALUE] LOCAL_PATH or https://BUCKET")
os.Exit(1)
}
loop := pmtiles.NewLoop(path, logger, *cacheSize, *cors)
loop, err := pmtiles.NewLoop(path, logger, *cacheSize, *cors)

if err != nil {
logger.Fatalf("Failed to create new loop, %v", err)
}

loop.Start()

http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
Expand Down Expand Up @@ -83,9 +92,19 @@ func main() {
convertCmd.Parse(os.Args[2:])
path := convertCmd.Arg(0)
output := convertCmd.Arg(1)
pmtiles.Convert(logger, path, output)
err := pmtiles.Convert(logger, path, output)

if err != nil {
logger.Fatalf("Failed to convert %s, %v", path, err)
}

case "upload":
pmtiles.Upload(logger, os.Args[2:])
err := pmtiles.Upload(logger, os.Args[2:])

if err != nil {
logger.Fatalf("Failed to upload file, %v", err)
}

case "validate":
// pmtiles.Validate()
default:
Expand Down
60 changes: 33 additions & 27 deletions pmtiles/convert.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"compress/gzip"
"encoding/json"
"errors"
"fmt"
"github.com/RoaringBitmap/roaring/roaring64"
"github.com/schollz/progressbar/v3"
"hash"
Expand Down Expand Up @@ -85,11 +86,11 @@ func NewResolver() *Resolver {
return &r
}

func Convert(logger *log.Logger, input string, output string) {
func Convert(logger *log.Logger, input string, output string) error {
if strings.HasSuffix(input, ".pmtiles") {
ConvertPmtilesV2(logger, input, output)
return ConvertPmtilesV2(logger, input, output)
} else {
ConvertMbtiles(logger, input, output)
return ConvertMbtiles(logger, input, output)
}
}

Expand All @@ -108,17 +109,17 @@ func add_directoryv2_entries(dir DirectoryV2, entries *[]EntryV3, f *os.File) {
}
}

func ConvertPmtilesV2(logger *log.Logger, input string, output string) {
func ConvertPmtilesV2(logger *log.Logger, input string, output string) error {
start := time.Now()
f, err := os.Open(input)
if err != nil {
log.Fatalf("Failed to open file: %s", err)
return fmt.Errorf("Failed to open file: %w", err)
}
defer f.Close()
buffer := make([]byte, 512000)
io.ReadFull(f, buffer)
if string(buffer[0:7]) == "PMTiles" && buffer[7] == 3 {
logger.Fatal("Archive is already the latest PMTiles version (3).")
return fmt.Errorf("Archive is already the latest PMTiles version (3).")
}

v2_json_bytes, dir := ParseHeaderV2(bytes.NewReader(buffer))
Expand All @@ -132,13 +133,13 @@ func ConvertPmtilesV2(logger *log.Logger, input string, output string) {
f.Seek(512000, 0)
n, err := f.Read(first4)
if n != 4 || err != nil {
panic(err)
return fmt.Errorf("Failed to read first 4, %w", err)
}

header, json_metadata, err := v2_to_header_json(v2_metadata, first4)

if err != nil {
logger.Fatal(err)
return fmt.Errorf("Failed to convert v2 to header JSON, %w", err)
}

entries := make([]EntryV3, 0)
Expand All @@ -151,7 +152,7 @@ func ConvertPmtilesV2(logger *log.Logger, input string, output string) {

tmpfile, err := ioutil.TempFile("", "")
if err != nil {
logger.Fatal(err)
return fmt.Errorf("Failed to create temp file, %w", err)
}
defer os.Remove(tmpfile.Name())

Expand All @@ -164,13 +165,13 @@ func ConvertPmtilesV2(logger *log.Logger, input string, output string) {
}
_, err := f.Seek(int64(entry.Offset), 0)
if err != nil {
panic(err)
return fmt.Errorf("Failed to seek at offset %d, %w", entry.Offset, err)
}
buf := make([]byte, entry.Length)
_, err = f.Read(buf)
if err != nil {
if err != io.EOF {
panic(err)
return fmt.Errorf("Failed to read buffer, %w", err)
}
}
if is_new, new_data := resolver.AddTileIsNew(entry.TileId, buf); is_new {
Expand All @@ -180,28 +181,30 @@ func ConvertPmtilesV2(logger *log.Logger, input string, output string) {

finalize(logger, resolver, header, tmpfile, output, json_metadata)
logger.Println("Finished in ", time.Since(start))

return nil
}

func ConvertMbtiles(logger *log.Logger, input string, output string) {
func ConvertMbtiles(logger *log.Logger, input string, output string) error {
start := time.Now()
conn, err := sqlite.OpenConn(input, sqlite.OpenReadOnly)
if err != nil {
logger.Fatal(err)
return fmt.Errorf("Failed to create database connection, %w", err)
}
defer conn.Close()

mbtiles_metadata := make([]string, 0)
{
stmt, _, err := conn.PrepareTransient("SELECT name, value FROM metadata")
if err != nil {
logger.Fatal(err)
return fmt.Errorf("Failed to create SQL statement, %w", err)
}
defer stmt.Finalize()

for {
row, err := stmt.Step()
if err != nil {
logger.Fatal(err)
return fmt.Errorf("Failed to step statement, %w", err)
}
if !row {
break
Expand All @@ -213,7 +216,7 @@ func ConvertMbtiles(logger *log.Logger, input string, output string) {
header, json_metadata, err := mbtiles_to_header_json(mbtiles_metadata)

if err != nil {
logger.Fatal(err)
return fmt.Errorf("Failed to convert MBTiles to header JSON, %w", err)
}

logger.Println("Querying total tile count...")
Expand All @@ -222,12 +225,12 @@ func ConvertMbtiles(logger *log.Logger, input string, output string) {
{
stmt, _, err := conn.PrepareTransient("SELECT count(*) FROM tiles")
if err != nil {
logger.Fatal(err)
return fmt.Errorf("Failed to create statement, %w", err)
}
defer stmt.Finalize()
row, err := stmt.Step()
if err != nil || !row {
logger.Fatal(err)
return fmt.Errorf("Failed to step row, %w", err)
}
total_tiles = stmt.ColumnInt64(0)
}
Expand All @@ -238,7 +241,7 @@ func ConvertMbtiles(logger *log.Logger, input string, output string) {
{
stmt, _, err := conn.PrepareTransient("SELECT zoom_level, tile_column, tile_row FROM tiles")
if err != nil {
logger.Fatal(err)
return fmt.Errorf("Failed to create statement, %w", err)
}
defer stmt.Finalize()

Expand All @@ -247,7 +250,7 @@ func ConvertMbtiles(logger *log.Logger, input string, output string) {
for {
row, err := stmt.Step()
if err != nil {
logger.Fatal(err)
return fmt.Errorf("Failed to step statement, %w", err)
}
if !row {
break
Expand All @@ -264,7 +267,7 @@ func ConvertMbtiles(logger *log.Logger, input string, output string) {

tmpfile, err := ioutil.TempFile("", "")
if err != nil {
logger.Fatal(err)
return fmt.Errorf("Failed to create temporary file, %w", err)
}
defer os.Remove(tmpfile.Name())

Expand All @@ -289,10 +292,10 @@ func ConvertMbtiles(logger *log.Logger, input string, output string) {

has_row, err := stmt.Step()
if err != nil {
logger.Fatal(err)
return fmt.Errorf("Failed to step statement, %w", err)
}
if !has_row {
logger.Fatal("Missing row")
return fmt.Errorf("Missing row")
}

reader := stmt.ColumnReader(0)
Expand All @@ -313,9 +316,10 @@ func ConvertMbtiles(logger *log.Logger, input string, output string) {
}
finalize(logger, resolver, header, tmpfile, output, json_metadata)
logger.Println("Finished in ", time.Since(start))
return nil
}

func finalize(logger *log.Logger, resolver *Resolver, header HeaderV3, tmpfile *os.File, output string, json_metadata map[string]interface{}) {
func finalize(logger *log.Logger, resolver *Resolver, header HeaderV3, tmpfile *os.File, output string, json_metadata map[string]interface{}) error {
logger.Println("# of addressed tiles: ", resolver.AddressedTiles)
logger.Println("# of tile entries (after RLE): ", len(resolver.Entries))
logger.Println("# of tile contents: ", len(resolver.OffsetMap))
Expand All @@ -327,7 +331,7 @@ func finalize(logger *log.Logger, resolver *Resolver, header HeaderV3, tmpfile *
// assemble the final file
outfile, err := os.Create(output)
if err != nil {
logger.Fatal(err)
return fmt.Errorf("Failed to create %s, %w", output, err)
}

root_bytes, leaves_bytes, num_leaves := optimize_directories(resolver.Entries, 16384-HEADERV3_LEN_BYTES)
Expand All @@ -348,7 +352,7 @@ func finalize(logger *log.Logger, resolver *Resolver, header HeaderV3, tmpfile *
{
metadata_bytes_uncompressed, err := json.Marshal(json_metadata)
if err != nil {
logger.Fatal(err)
return fmt.Errorf("Failed to marshal metadata, %w", err)
}
var b bytes.Buffer
w, _ := gzip.NewWriterLevel(&b, gzip.BestCompression)
Expand Down Expand Up @@ -379,8 +383,10 @@ func finalize(logger *log.Logger, resolver *Resolver, header HeaderV3, tmpfile *
tmpfile.Seek(0, 0)
_, err = io.Copy(outfile, tmpfile)
if err != nil {
logger.Fatal(err)
return fmt.Errorf("Failed to copy data to outfile, %w", err)
}

return nil
}

func v2_to_header_json(v2_json_metadata map[string]interface{}, first4 []byte) (HeaderV3, map[string]interface{}, error) {
Expand Down
6 changes: 3 additions & 3 deletions pmtiles/convert_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,9 +90,9 @@ func TestV2UpgradeBarebones(t *testing.T) {

func TestV2UpgradeStrings(t *testing.T) {
header, _, err := v2_to_header_json(map[string]interface{}{
"minzoom": "0",
"maxzoom": "14",
"bounds": "-180.0,-85,178,83",
"minzoom": "0",
"maxzoom": "14",
"bounds": "-180.0,-85,178,83",
}, []byte{0x1f, 0x8b, 0x0, 0x0})
if err != nil {
t.Fatalf("parsing error %s", err)
Expand Down
68 changes: 34 additions & 34 deletions pmtiles/directory_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,9 @@ package pmtiles

import (
"bytes"
"github.com/stretchr/testify/assert"
"math/rand"
"testing"
"github.com/stretchr/testify/assert"
)

func TestDirectoryRoundtrip(t *testing.T) {
Expand Down Expand Up @@ -188,58 +188,58 @@ func TestOptimizeDirectories(t *testing.T) {
}

func TestFindTileMissing(t *testing.T) {
entries := make([]EntryV3,0)
_, ok := find_tile(entries,0)
entries := make([]EntryV3, 0)
_, ok := find_tile(entries, 0)
if ok {
t.Fatalf("Expected not ok")
}
}

func TestFindTileFirstEntry(t *testing.T) {
entries := []EntryV3{{TileId:100, Offset: 1, Length: 1, RunLength:1}}
entry, ok := find_tile(entries,100)
assert.Equal(t,true,ok)
assert.Equal(t,uint64(1),entry.Offset)
assert.Equal(t,uint32(1),entry.Length)
_, ok = find_tile(entries,101)
assert.Equal(t,false,ok)
entries := []EntryV3{{TileId: 100, Offset: 1, Length: 1, RunLength: 1}}
entry, ok := find_tile(entries, 100)
assert.Equal(t, true, ok)
assert.Equal(t, uint64(1), entry.Offset)
assert.Equal(t, uint32(1), entry.Length)
_, ok = find_tile(entries, 101)
assert.Equal(t, false, ok)
}

func TestFindTileMultipleEntries(t *testing.T) {
entries := []EntryV3{
{TileId:100, Offset: 1, Length: 1, RunLength:2},
{TileId: 100, Offset: 1, Length: 1, RunLength: 2},
}
entry, ok := find_tile(entries,101)
assert.Equal(t,true,ok)
assert.Equal(t,uint64(1),entry.Offset)
assert.Equal(t,uint32(1),entry.Length)
entry, ok := find_tile(entries, 101)
assert.Equal(t, true, ok)
assert.Equal(t, uint64(1), entry.Offset)
assert.Equal(t, uint32(1), entry.Length)

entries = []EntryV3{
{TileId:100, Offset: 1, Length: 1, RunLength:1},
{TileId:150, Offset: 2, Length: 2, RunLength:2},
{TileId: 100, Offset: 1, Length: 1, RunLength: 1},
{TileId: 150, Offset: 2, Length: 2, RunLength: 2},
}
entry, ok = find_tile(entries,151)
assert.Equal(t,true,ok)
assert.Equal(t,uint64(2),entry.Offset)
assert.Equal(t,uint32(2),entry.Length)
entry, ok = find_tile(entries, 151)
assert.Equal(t, true, ok)
assert.Equal(t, uint64(2), entry.Offset)
assert.Equal(t, uint32(2), entry.Length)

entries = []EntryV3{
{TileId:50, Offset: 1, Length: 1, RunLength:2},
{TileId:100, Offset: 2, Length: 2, RunLength:1},
{TileId:150, Offset: 3, Length: 3, RunLength:1},
}
entry, ok = find_tile(entries,51)
assert.Equal(t,true,ok)
assert.Equal(t,uint64(1),entry.Offset)
assert.Equal(t,uint32(1),entry.Length)
{TileId: 50, Offset: 1, Length: 1, RunLength: 2},
{TileId: 100, Offset: 2, Length: 2, RunLength: 1},
{TileId: 150, Offset: 3, Length: 3, RunLength: 1},
}
entry, ok = find_tile(entries, 51)
assert.Equal(t, true, ok)
assert.Equal(t, uint64(1), entry.Offset)
assert.Equal(t, uint32(1), entry.Length)
}

func TestFindTileLeafSearch(t *testing.T) {
entries := []EntryV3{
{TileId:100, Offset: 1, Length: 1, RunLength:0},
{TileId: 100, Offset: 1, Length: 1, RunLength: 0},
}
entry, ok := find_tile(entries,150)
assert.Equal(t,true,ok)
assert.Equal(t,uint64(1),entry.Offset)
assert.Equal(t,uint32(1),entry.Length)
entry, ok := find_tile(entries, 150)
assert.Equal(t, true, ok)
assert.Equal(t, uint64(1), entry.Offset)
assert.Equal(t, uint32(1), entry.Length)
}
Loading

0 comments on commit 0493afb

Please sign in to comment.