Runtime: Retrofit metricsview package to MetricsView RPCs (#5148)

* Retrofit MVC * Migrate totals, toplist, timeseries * Fix tests * Fix test * Review
rilldata · Jun 27, 2024 · 208ede5 · 208ede5 · github-actions · Jun 27, 2024
1 parent a81b038
commit 208ede5
Show file tree

Hide file tree

Showing 38 changed files with 2,027 additions and 3,890 deletions.
diff --git a/proto/gen/rill/admin/v1/admin.swagger.yaml b/proto/gen/rill/admin/v1/admin.swagger.yaml
@@ -2432,7 +2432,7 @@ definitions:
       `NullValue` is a singleton enumeration to represent the null value for the
       `Value` type union.
 
-      The JSON representation for `NullValue` is JSON `null`.
+       The JSON representation for `NullValue` is JSON `null`.
 
        - NULL_VALUE: Null value.
   rpcStatus:

diff --git a/proto/gen/rill/runtime/v1/queries.pb.go b/proto/gen/rill/runtime/v1/queries.pb.go
diff --git a/proto/gen/rill/runtime/v1/queries.pb.validate.go b/proto/gen/rill/runtime/v1/queries.pb.validate.go
diff --git a/proto/gen/rill/runtime/v1/runtime.swagger.yaml b/proto/gen/rill/runtime/v1/runtime.swagger.yaml
@@ -1553,12 +1553,6 @@ paths:
                 type: array
                 items:
                   type: string
-                title: Required either measure_names or inline_measures
-              inlineMeasures:
-                type: array
-                items:
-                  type: object
-                  $ref: '#/definitions/v1InlineMeasure'
               timeStart:
                 type: string
                 format: date-time
@@ -1624,11 +1618,6 @@ paths:
                 type: array
                 items:
                   type: string
-              inlineMeasures:
-                type: array
-                items:
-                  type: object
-                  $ref: '#/definitions/v1InlineMeasure'
               timeStart:
                 type: string
                 format: date-time
@@ -1693,12 +1682,6 @@ paths:
                 type: array
                 items:
                   type: string
-                title: Required either measure_names or inline_measures
-              inlineMeasures:
-                type: array
-                items:
-                  type: object
-                  $ref: '#/definitions/v1InlineMeasure'
               timeStart:
                 type: string
                 format: date-time
@@ -2931,7 +2914,7 @@ definitions:
       `NullValue` is a singleton enumeration to represent the null value for the
       `Value` type union.
 
-      The JSON representation for `NullValue` is JSON `null`.
+       The JSON representation for `NullValue` is JSON `null`.
 
        - NULL_VALUE: Null value.
   rpcStatus:
@@ -3891,15 +3874,6 @@ definitions:
       - HISTOGRAM_METHOD_FD
       - HISTOGRAM_METHOD_DIAGNOSTIC
     default: HISTOGRAM_METHOD_UNSPECIFIED
-  v1InlineMeasure:
-    type: object
-    properties:
-      name:
-        type: string
-        title: Required
-      expression:
-        type: string
-        title: Required, ie 'count(*)'
   v1Instance:
     type: object
     properties:
@@ -4569,12 +4543,6 @@ definitions:
         type: array
         items:
           type: string
-        title: Required either measure_names or inline_measures
-      inlineMeasures:
-        type: array
-        items:
-          type: object
-          $ref: '#/definitions/v1InlineMeasure'
       timeStart:
         type: string
         format: date-time
@@ -4629,11 +4597,6 @@ definitions:
         type: array
         items:
           type: string
-      inlineMeasures:
-        type: array
-        items:
-          type: object
-          $ref: '#/definitions/v1InlineMeasure'
       timeStart:
         type: string
         format: date-time
@@ -4687,12 +4650,6 @@ definitions:
         type: array
         items:
           type: string
-        title: Required either measure_names or inline_measures
-      inlineMeasures:
-        type: array
-        items:
-          type: object
-          $ref: '#/definitions/v1InlineMeasure'
       timeStart:
         type: string
         format: date-time

diff --git a/proto/rill/runtime/v1/queries.proto b/proto/rill/runtime/v1/queries.proto
@@ -429,7 +429,6 @@ message MetricsViewToplistRequest {
   string metrics_view_name = 2 [(validate.rules).string.min_len = 1];
   string dimension_name = 3 [(validate.rules).string.min_len = 1];
   repeated string measure_names = 4;
-  repeated InlineMeasure inline_measures = 12;
   google.protobuf.Timestamp time_start = 5;
   google.protobuf.Timestamp time_end = 6;
   int64 limit = 7 [(validate.rules).int64.gte = 0];
@@ -559,9 +558,7 @@ message MetricsViewComparisonMeasureAlias {
 message MetricsViewTimeSeriesRequest {
   string instance_id = 1;
   string metrics_view_name = 2 [(validate.rules).string.min_len = 1];
-  // Required either measure_names or inline_measures
   repeated string measure_names = 3 [(validate.rules).repeated.min_items = 1];
-  repeated InlineMeasure inline_measures = 9;
   // Optional. Defaults to min
   google.protobuf.Timestamp time_start = 4;
   // Optional. Defaults to max
@@ -590,9 +587,7 @@ message MetricsViewTimeSeriesResponse {
 message MetricsViewTotalsRequest {
   string instance_id = 1;
   string metrics_view_name = 2 [(validate.rules).string.min_len = 1];
-  // Required either measure_names or inline_measures
   repeated string measure_names = 3 [(validate.rules).repeated.min_items = 1];
-  repeated InlineMeasure inline_measures = 9;
   // Optional. Defaults to min
   google.protobuf.Timestamp time_start = 4;
   // Optional. Defaults to max

diff --git a/runtime/drivers/duckdb/model_executor_self_file.go b/runtime/drivers/duckdb/model_executor_self_file.go
@@ -72,22 +72,22 @@ func (e *selfToFileExecutor) Execute(ctx context.Context) (*drivers.ModelResult,
 	}, nil
 }
 
-func exportSQL(qry, path, format string) (string, error) {
+func exportSQL(qry, path string, format drivers.FileFormat) (string, error) {
 	switch format {
-	case "parquet":
+	case drivers.FileFormatParquet:
 		return fmt.Sprintf("COPY (%s\n) TO '%s' (FORMAT PARQUET)", qry, path), nil
-	case "csv":
+	case drivers.FileFormatCSV:
 		return fmt.Sprintf("COPY (%s\n) TO '%s' (FORMAT CSV, HEADER true)", qry, path), nil
-	case "json":
+	case drivers.FileFormatJSON:
 		return fmt.Sprintf("COPY (%s\n) TO '%s' (FORMAT JSON)", qry, path), nil
 	default:
 		return "", fmt.Errorf("duckdb: unsupported export format %q", format)
 	}
 }
 
-func supportsExportFormat(format string) bool {
+func supportsExportFormat(format drivers.FileFormat) bool {
 	switch format {
-	case "parquet", "csv", "json":
+	case drivers.FileFormatParquet, drivers.FileFormatCSV, drivers.FileFormatJSON:
 		return true
 	default:
 		return false

diff --git a/runtime/drivers/file/model_executor.go b/runtime/drivers/file/model_executor.go
@@ -1,10 +1,14 @@
 package file
 
-import "fmt"
+import (
+	"fmt"
+
+	"github.com/rilldata/rill/runtime/drivers"
+)
 
 type ModelOutputProperties struct {
-	Path   string `mapstructure:"path"`
-	Format string `mapstructure:"format"`
+	Path   string             `mapstructure:"path"`
+	Format drivers.FileFormat `mapstructure:"format"`
 }
 
 func (p *ModelOutputProperties) Validate() error {
@@ -13,11 +17,13 @@ func (p *ModelOutputProperties) Validate() error {
 	}
 	if p.Format == "" {
 		return fmt.Errorf("missing property 'format'")
+	} else if !p.Format.Valid() {
+		return fmt.Errorf("invalid property 'format': %q", p.Format)
 	}
 	return nil
 }
 
 type ModelResultProperties struct {
-	Path   string `mapstructure:"path"`
-	Format string `mapstructure:"format"`
+	Path   string             `mapstructure:"path"`
+	Format drivers.FileFormat `mapstructure:"format"`
 }
diff --git a/runtime/drivers/file/model_executor_olap_self.go b/runtime/drivers/file/model_executor_olap_self.go
@@ -62,12 +62,14 @@ func (e *olapToSelfExecutor) Execute(ctx context.Context) (*drivers.ModelResult,
 	defer res.Close()
 
 	switch outputProps.Format {
-	case "csv":
+	case drivers.FileFormatParquet:
+		err = writeParquet(res, outputProps.Path)
+	case drivers.FileFormatCSV:
 		err = writeCSV(res, outputProps.Path)
-	case "xlsx":
+	case drivers.FileFormatJSON:
+		return nil, errors.New("json file output not currently supported")
+	case drivers.FileFormatXLSX:
 		err = writeXLSX(res, outputProps.Path)
-	case "parquet":
-		err = writeParquet(res, outputProps.Path)
 	default:
 		return nil, fmt.Errorf("unsupported output format %q", outputProps.Format)
 	}

diff --git a/runtime/drivers/models.go b/runtime/drivers/models.go
@@ -40,3 +40,25 @@ type ModelExecutorOptions struct {
 	IncrementalRun   bool
 	PreviousResult   *ModelResult
 }
+
+type FileFormat string
+
+const (
+	FileFormatUnspecified FileFormat = ""
+	FileFormatParquet     FileFormat = "parquet"
+	FileFormatCSV         FileFormat = "csv"
+	FileFormatJSON        FileFormat = "json"
+	FileFormatXLSX        FileFormat = "xlsx"
+)
+
+func (f FileFormat) Filename(stem string) string {
+	return stem + "." + string(f)
+}
+
+func (f FileFormat) Valid() bool {
+	switch f {
+	case FileFormatParquet, FileFormatCSV, FileFormatJSON, FileFormatXLSX:
+		return true
+	}
+	return false
+}