upload and serve allow S3 safe keys and prefixes [#19]

pmtiles/convert.go

@@ -145,7 +145,7 @@ func ConvertPmtilesV2(logger *log.Logger, input string, output string) error {
 	entries := make([]EntryV3, 0)
 	add_directoryv2_entries(dir, &entries, f)
 
-	// sort + RLE encoding
+	// sort
 	sort.Slice(entries, func(i, j int) bool {
 		return entries[i].TileId < entries[j].TileId
 	})
@@ -174,6 +174,7 @@ func ConvertPmtilesV2(logger *log.Logger, input string, output string) error {
 				return fmt.Errorf("Failed to read buffer, %w", err)
 			}
 		}
+		// TODO: enforce sorted order
 		if is_new, new_data := resolver.AddTileIsNew(entry.TileId, buf); is_new {
 			tmpfile.Write(new_data)
 		}
@@ -590,7 +591,7 @@ func mbtiles_to_header_json(mbtiles_metadata []string) (HeaderV3, map[string]int
 		case "compression":
 			switch value {
 			case "gzip":
-				header.TileCompression = Gzip
+				header.TileCompression = Gzip // TODO: fix me for non-vector outputs
 			}
 			json_result["compression"] = value
 		// name, attribution, description, type, version

pmtiles/loop.go

@@ -292,25 +292,40 @@ func (loop *Loop) get_tile(ctx context.Context, http_headers map[string]string,
 	return 204, http_headers, nil
 }
 
-var tilePattern = regexp.MustCompile(`.*?\/([-A-Za-z0-9_]+)\/(\d+)\/(\d+)\/(\d+)\.([a-z]+)$`)
-var metadataPattern = regexp.MustCompile(`.*?\/([-A-Za-z0-9_]+)\/metadata$`)
+var tilePattern = regexp.MustCompile(`^\/([-A-Za-z0-9_\/!-_\.\*'\(\)']+)\/(\d+)\/(\d+)\/(\d+)\.([a-z]+)$`)
+var metadataPattern = regexp.MustCompile(`^\/([-A-Za-z0-9_\/!-_\.\*'\(\)']+)\/metadata$`)
 
-func (loop *Loop) Get(ctx context.Context, path string) (int, map[string]string, []byte) {
-	http_headers := make(map[string]string)
-	if len(loop.cors) > 0 {
-		http_headers["Access-Control-Allow-Origin"] = loop.cors
-	}
+func parse_tile_path(path string) (bool, string, uint8, uint32, uint32, string) {
 	if res := tilePattern.FindStringSubmatch(path); res != nil {
 		name := res[1]
 		z, _ := strconv.ParseUint(res[2], 10, 8)
 		x, _ := strconv.ParseUint(res[3], 10, 32)
 		y, _ := strconv.ParseUint(res[4], 10, 32)
 		ext := res[5]
-		return loop.get_tile(ctx, http_headers, name, uint8(z), uint32(x), uint32(y), ext)
+		return true, name, uint8(z), uint32(x), uint32(y), ext
 	}
+	return false, "", 0, 0, 0, ""
+}
+
+func parse_metadata_path(path string) (bool, string) {
 	if res := metadataPattern.FindStringSubmatch(path); res != nil {
 		name := res[1]
-		return loop.get_metadata(ctx, http_headers, name)
+		return true, name
+	}
+	return false, ""
+}
+
+func (loop *Loop) Get(ctx context.Context, path string) (int, map[string]string, []byte) {
+	http_headers := make(map[string]string)
+	if len(loop.cors) > 0 {
+		http_headers["Access-Control-Allow-Origin"] = loop.cors
+	}
+
+	if ok, key, z, x, y, ext := parse_tile_path(path); ok {
+		return loop.get_tile(ctx, http_headers, key, z, x, y, ext)
+	}
+	if ok, key := parse_metadata_path(path); ok {
+		return loop.get_metadata(ctx, http_headers, key)
 	}
 
 	return 404, http_headers, []byte("Tile not found")

pmtiles/loop_test.go

@@ -0,0 +1,36 @@
+package pmtiles
+
+import (
+	"github.com/stretchr/testify/assert"
+	"testing"
+)
+
+func TestRegex(t *testing.T) {
+	ok, key, z, x, y, ext := parse_tile_path("/foo/0/0/0")
+	assert.False(t, ok)
+	ok, key, z, x, y, ext = parse_tile_path("/foo/0/0/0.mvt")
+	assert.True(t, ok)
+	assert.Equal(t, key, "foo")
+	assert.Equal(t, z, uint8(0))
+	assert.Equal(t, x, uint32(0))
+	assert.Equal(t, y, uint32(0))
+	assert.Equal(t, ext, "mvt")
+	ok, key, z, x, y, ext = parse_tile_path("/foo/bar/0/0/0.mvt")
+	assert.True(t, ok)
+	assert.Equal(t, key, "foo/bar")
+	assert.Equal(t, z, uint8(0))
+	assert.Equal(t, x, uint32(0))
+	assert.Equal(t, y, uint32(0))
+	assert.Equal(t, ext, "mvt")
+	// https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html
+	ok, key, z, x, y, ext = parse_tile_path("/!-_.*'()/0/0/0.mvt")
+	assert.True(t, ok)
+	assert.Equal(t, key, "!-_.*'()")
+	assert.Equal(t, z, uint8(0))
+	assert.Equal(t, x, uint32(0))
+	assert.Equal(t, y, uint32(0))
+	assert.Equal(t, ext, "mvt")
+	ok, key = parse_metadata_path("/!-_.*'()/metadata")
+	assert.True(t, ok)
+	assert.Equal(t, key, "!-_.*'()")
+}

pmtiles/upload.go

@@ -17,22 +17,23 @@ func Upload(logger *log.Logger, args []string) error {
 	max_concurrency := cmd.Int("max-concurrency", 5, "Number of upload threads")
 
 	cmd.Parse(args)
-	file := cmd.Arg(0)
+	source := cmd.Arg(0)
 	bucketURL := cmd.Arg(1)
+	destination := cmd.Arg(2)
 
-	if file == "" || bucketURL == "" {
-		return fmt.Errorf("USAGE: upload [-buffer-size B] [-max-concurrency M] INPUT s3://BUCKET?region=region")
+	if source == "" || bucketURL == "" || destination == "" {
+		return fmt.Errorf("USAGE: upload [-buffer-size B] [-max-concurrency M] INPUT s3://BUCKET?region=region DESTINATION")
 	}
 
-	logger.Println(file, bucketURL)
+	logger.Println(source, bucketURL, destination)
 	ctx := context.Background()
 	b, err := blob.OpenBucket(ctx, bucketURL)
 	if err != nil {
 		return fmt.Errorf("Failed to setup bucket: %w", err)
 	}
 	defer b.Close()
 
-	f, err := os.Open(file)
+	f, err := os.Open(source)
 	if err != nil {
 		return fmt.Errorf("Failed to open file: %w", err)
 	}
@@ -51,7 +52,7 @@ func Upload(logger *log.Logger, args []string) error {
 		MaxConcurrency: *max_concurrency,
 	}
 
-	w, err := b.NewWriter(ctx, file, opts)
+	w, err := b.NewWriter(ctx, destination, opts)
 	if err != nil {
 		return fmt.Errorf("Failed to obtain writer: %w", err)
 	}