From c2af73642cadd35d2147f0550b637c91a397b9e6 Mon Sep 17 00:00:00 2001 From: Brandon Liu Date: Thu, 13 Oct 2022 11:12:35 +0800 Subject: [PATCH] upload and serve allow S3 safe keys and prefixes [#19] --- pmtiles/convert.go | 5 +++-- pmtiles/loop.go | 33 ++++++++++++++++++++++++--------- pmtiles/loop_test.go | 36 ++++++++++++++++++++++++++++++++++++ pmtiles/upload.go | 13 +++++++------ 4 files changed, 70 insertions(+), 17 deletions(-) create mode 100644 pmtiles/loop_test.go diff --git a/pmtiles/convert.go b/pmtiles/convert.go index 505d658..0b22cb6 100644 --- a/pmtiles/convert.go +++ b/pmtiles/convert.go @@ -145,7 +145,7 @@ func ConvertPmtilesV2(logger *log.Logger, input string, output string) error { entries := make([]EntryV3, 0) add_directoryv2_entries(dir, &entries, f) - // sort + RLE encoding + // sort sort.Slice(entries, func(i, j int) bool { return entries[i].TileId < entries[j].TileId }) @@ -174,6 +174,7 @@ func ConvertPmtilesV2(logger *log.Logger, input string, output string) error { return fmt.Errorf("Failed to read buffer, %w", err) } } + // TODO: enforce sorted order if is_new, new_data := resolver.AddTileIsNew(entry.TileId, buf); is_new { tmpfile.Write(new_data) } @@ -590,7 +591,7 @@ func mbtiles_to_header_json(mbtiles_metadata []string) (HeaderV3, map[string]int case "compression": switch value { case "gzip": - header.TileCompression = Gzip + header.TileCompression = Gzip // TODO: fix me for non-vector outputs } json_result["compression"] = value // name, attribution, description, type, version diff --git a/pmtiles/loop.go b/pmtiles/loop.go index feab5f5..f639dfe 100644 --- a/pmtiles/loop.go +++ b/pmtiles/loop.go @@ -292,25 +292,40 @@ func (loop *Loop) get_tile(ctx context.Context, http_headers map[string]string, return 204, http_headers, nil } -var tilePattern = regexp.MustCompile(`.*?\/([-A-Za-z0-9_]+)\/(\d+)\/(\d+)\/(\d+)\.([a-z]+)$`) -var metadataPattern = regexp.MustCompile(`.*?\/([-A-Za-z0-9_]+)\/metadata$`) +var tilePattern = regexp.MustCompile(`^\/([-A-Za-z0-9_\/!-_\.\*'\(\)']+)\/(\d+)\/(\d+)\/(\d+)\.([a-z]+)$`) +var metadataPattern = regexp.MustCompile(`^\/([-A-Za-z0-9_\/!-_\.\*'\(\)']+)\/metadata$`) -func (loop *Loop) Get(ctx context.Context, path string) (int, map[string]string, []byte) { - http_headers := make(map[string]string) - if len(loop.cors) > 0 { - http_headers["Access-Control-Allow-Origin"] = loop.cors - } +func parse_tile_path(path string) (bool, string, uint8, uint32, uint32, string) { if res := tilePattern.FindStringSubmatch(path); res != nil { name := res[1] z, _ := strconv.ParseUint(res[2], 10, 8) x, _ := strconv.ParseUint(res[3], 10, 32) y, _ := strconv.ParseUint(res[4], 10, 32) ext := res[5] - return loop.get_tile(ctx, http_headers, name, uint8(z), uint32(x), uint32(y), ext) + return true, name, uint8(z), uint32(x), uint32(y), ext } + return false, "", 0, 0, 0, "" +} + +func parse_metadata_path(path string) (bool, string) { if res := metadataPattern.FindStringSubmatch(path); res != nil { name := res[1] - return loop.get_metadata(ctx, http_headers, name) + return true, name + } + return false, "" +} + +func (loop *Loop) Get(ctx context.Context, path string) (int, map[string]string, []byte) { + http_headers := make(map[string]string) + if len(loop.cors) > 0 { + http_headers["Access-Control-Allow-Origin"] = loop.cors + } + + if ok, key, z, x, y, ext := parse_tile_path(path); ok { + return loop.get_tile(ctx, http_headers, key, z, x, y, ext) + } + if ok, key := parse_metadata_path(path); ok { + return loop.get_metadata(ctx, http_headers, key) } return 404, http_headers, []byte("Tile not found") diff --git a/pmtiles/loop_test.go b/pmtiles/loop_test.go new file mode 100644 index 0000000..d0bcaf2 --- /dev/null +++ b/pmtiles/loop_test.go @@ -0,0 +1,36 @@ +package pmtiles + +import ( + "github.com/stretchr/testify/assert" + "testing" +) + +func TestRegex(t *testing.T) { + ok, key, z, x, y, ext := parse_tile_path("/foo/0/0/0") + assert.False(t, ok) + ok, key, z, x, y, ext = parse_tile_path("/foo/0/0/0.mvt") + assert.True(t, ok) + assert.Equal(t, key, "foo") + assert.Equal(t, z, uint8(0)) + assert.Equal(t, x, uint32(0)) + assert.Equal(t, y, uint32(0)) + assert.Equal(t, ext, "mvt") + ok, key, z, x, y, ext = parse_tile_path("/foo/bar/0/0/0.mvt") + assert.True(t, ok) + assert.Equal(t, key, "foo/bar") + assert.Equal(t, z, uint8(0)) + assert.Equal(t, x, uint32(0)) + assert.Equal(t, y, uint32(0)) + assert.Equal(t, ext, "mvt") + // https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html + ok, key, z, x, y, ext = parse_tile_path("/!-_.*'()/0/0/0.mvt") + assert.True(t, ok) + assert.Equal(t, key, "!-_.*'()") + assert.Equal(t, z, uint8(0)) + assert.Equal(t, x, uint32(0)) + assert.Equal(t, y, uint32(0)) + assert.Equal(t, ext, "mvt") + ok, key = parse_metadata_path("/!-_.*'()/metadata") + assert.True(t, ok) + assert.Equal(t, key, "!-_.*'()") +} diff --git a/pmtiles/upload.go b/pmtiles/upload.go index 51f44fc..2fe9e22 100644 --- a/pmtiles/upload.go +++ b/pmtiles/upload.go @@ -17,14 +17,15 @@ func Upload(logger *log.Logger, args []string) error { max_concurrency := cmd.Int("max-concurrency", 5, "Number of upload threads") cmd.Parse(args) - file := cmd.Arg(0) + source := cmd.Arg(0) bucketURL := cmd.Arg(1) + destination := cmd.Arg(2) - if file == "" || bucketURL == "" { - return fmt.Errorf("USAGE: upload [-buffer-size B] [-max-concurrency M] INPUT s3://BUCKET?region=region") + if source == "" || bucketURL == "" || destination == "" { + return fmt.Errorf("USAGE: upload [-buffer-size B] [-max-concurrency M] INPUT s3://BUCKET?region=region DESTINATION") } - logger.Println(file, bucketURL) + logger.Println(source, bucketURL, destination) ctx := context.Background() b, err := blob.OpenBucket(ctx, bucketURL) if err != nil { @@ -32,7 +33,7 @@ func Upload(logger *log.Logger, args []string) error { } defer b.Close() - f, err := os.Open(file) + f, err := os.Open(source) if err != nil { return fmt.Errorf("Failed to open file: %w", err) } @@ -51,7 +52,7 @@ func Upload(logger *log.Logger, args []string) error { MaxConcurrency: *max_concurrency, } - w, err := b.NewWriter(ctx, file, opts) + w, err := b.NewWriter(ctx, destination, opts) if err != nil { return fmt.Errorf("Failed to obtain writer: %w", err) }