Merge pull request #12 from mullerpeter/feature/type-reflection

✨ feat: support all databricks data types

CHANGELOG.md

@@ -1,7 +1,13 @@
 # Changelog
-## 1.0.0
 
-### Breaking Changes: 
+### 1.1.0
+
+- Adds proper type reflection to support all databricks data types. (except type `BINARY` which is not supported)
+---
+
+### 1.0.0
+
+#### Breaking Changes: 
 
 - Datasource configuration (`hostname` & `sql path`) needs to be entered again in the Datasource Settings after upgrading.
 - Queries made via Visual Editor no longer work.
@@ -13,7 +19,6 @@
 - Made some config vars non secret
 - Added alerting capability
 
----
 
 ### 0.0.9
 

go.mod

@@ -31,7 +31,6 @@ require (
 	github.com/hashicorp/go-hclog v0.14.1 // indirect
 	github.com/hashicorp/go-plugin v1.4.3 // indirect
 	github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d // indirect
-	github.com/jmoiron/sqlx v1.3.5 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/klauspost/compress v1.13.1 // indirect
@@ -41,7 +40,6 @@ require (
 	github.com/mattn/go-colorable v0.1.4 // indirect
 	github.com/mattn/go-isatty v0.0.10 // indirect
 	github.com/mattn/go-runewidth v0.0.9 // indirect
-	github.com/mattn/go-sqlite3 v1.14.16 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
 	github.com/mitchellh/go-testing-interface v1.0.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect

go.sum

@@ -112,7 +112,6 @@ github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34
 github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
 github.com/go-openapi/swag v0.19.15 h1:D2NRCBzS9/pEY3gP9Nl8aDqGUcPFrwG2p+CNFrLyrCM=
 github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=
-github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
@@ -200,8 +199,6 @@ github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d/go.mod h1:+NfK9FKe
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/jhump/protoreflect v1.6.0 h1:h5jfMVslIg6l29nsMs0D8Wj17RDVdNYti0vDN/PZZoE=
 github.com/jhump/protoreflect v1.6.0/go.mod h1:eaTn3RZAmMBcV0fifFvlm6VHNz3wSkYyXYWUh7ymB74=
-github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g=
-github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
@@ -228,7 +225,6 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
-github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/magefile/mage v1.13.0 h1:XtLJl8bcCM7EFoO8FyH8XK3t7G5hQAeK+i4tq+veT9M=
 github.com/magefile/mage v1.13.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A=
 github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
@@ -245,9 +241,6 @@ github.com/mattn/go-isatty v0.0.10 h1:qxFzApOv4WsAL965uUPIsXzAKCZxN2p9UqdhFS4ZW1
 github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84=
 github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0=
 github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
-github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
-github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y=
-github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
 github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=

package.json

@@ -1,7 +1,7 @@
 {
   "name": "mullerpeter-databricks-datasource",
   "private": true,
-  "version": "1.0.0",
+  "version": "1.1.0",
   "description": "Databricks SQL Connector",
   "scripts": {
     "build": "grafana-toolkit plugin:build",

pkg/plugin/plugin.go

@@ -11,7 +11,8 @@ import (
 	"github.com/grafana/grafana-plugin-sdk-go/backend/instancemgmt"
 	"github.com/grafana/grafana-plugin-sdk-go/backend/log"
 	"github.com/grafana/grafana-plugin-sdk-go/data"
-	"github.com/jmoiron/sqlx"
+	"github.com/grafana/grafana-plugin-sdk-go/data/sqlutil"
+	"reflect"
 	"strings"
 	"time"
 )
@@ -31,7 +32,7 @@ var (
 	_                           backend.StreamHandler         = (*SampleDatasource)(nil)
 	_                           instancemgmt.InstanceDisposer = (*SampleDatasource)(nil)
 	databricksConnectionsString string
-	databricksDB                *sqlx.DB
+	databricksDB                *sql.DB
 )
 
 type DatasourceSettings struct {
@@ -53,7 +54,7 @@ func NewSampleDatasource(settings backend.DataSourceInstanceSettings) (instancem
 		if err != nil {
 			log.DefaultLogger.Info("DB Init Error", "err", err)
 		} else {
-			databricksDB = sqlx.NewDb(db, "databricks")
+			databricksDB = db
 			log.DefaultLogger.Info("Store Databricks SQL DB Connection")
 		}
 	}
@@ -69,7 +70,7 @@ func RefreshDBConnection() error {
 			log.DefaultLogger.Info("DB Init Error", "err", err)
 			return err
 		} else {
-			databricksDB = sqlx.NewDb(db, "databricks")
+			databricksDB = db
 			log.DefaultLogger.Info("Store Databricks SQL DB Connection")
 			return nil
 		}
@@ -78,15 +79,15 @@ func RefreshDBConnection() error {
 	return errors.New("no connection string set")
 }
 
-func ExecuteQueryx(queryString string) (*sqlx.Rows, error) {
-	rows, err := databricksDB.Queryx(queryString)
+func ExecuteQuery(queryString string) (*sql.Rows, error) {
+	rows, err := databricksDB.Query(queryString)
 	if err != nil {
 		if strings.HasPrefix(err.Error(), "Invalid SessionHandle") {
 			err = RefreshDBConnection()
 			if err != nil {
 				return nil, err
 			}
-			rows, err = databricksDB.Queryx(queryString)
+			rows, err = databricksDB.Query(queryString)
 			if err != nil {
 				return nil, err
 			}
@@ -162,43 +163,43 @@ func (d *SampleDatasource) query(_ context.Context, pCtx backend.PluginContext,
 
 	frame := data.NewFrame("response")
 
-	rows, err := ExecuteQueryx(queryString)
+	rows, err := ExecuteQuery(queryString)
 	if err != nil {
 		response.Error = err
 		log.DefaultLogger.Info("Error", "err", err)
 		return response
 	}
 
-	colTypes, err := rows.ColumnTypes()
-
-	if err != nil {
-		response.Error = err
-		log.DefaultLogger.Info("Error", "err", err)
-		return response
-	}
-	columnNames, err := rows.Columns()
-	if err != nil {
-		response.Error = err
-		log.DefaultLogger.Info("Error", "err", err)
-		return response
+	dateConverter := sqlutil.Converter{
+		Name:          "Databricks date to timestamp converter",
+		InputScanType: reflect.TypeOf(sql.NullString{}),
+		InputTypeName: "DATE",
+		FrameConverter: sqlutil.FrameConverter{
+			FieldType: data.FieldTypeNullableTime,
+			ConverterFunc: func(n interface{}) (interface{}, error) {
+				v := n.(*sql.NullString)
+
+				if !v.Valid {
+					return (*time.Time)(nil), nil
+				}
+
+				f := v.String
+				date, error := time.Parse("2006-01-02", f)
+				if error != nil {
+					return (*time.Time)(nil), error
+				}
+				return &date, nil
+			},
+		},
 	}
 
-	frame, err = initDataframeTypes(colTypes, columnNames, frame)
+	frame, err = sqlutil.FrameFromRows(rows, -1, dateConverter)
 	if err != nil {
+		log.DefaultLogger.Info("FrameFromRows", "err", err)
 		response.Error = err
 		return response
 	}
 
-	for rows.Next() {
-		res, err := rows.SliceScan()
-		if err != nil {
-			response.Error = err
-			log.DefaultLogger.Info("Error", "err", err)
-			return response
-		}
-		frame.AppendRow(res...)
-	}
-
 	if qm.QuerySettings.ConvertLongToWide {
 		wideFrame, err := data.LongToWide(frame, &data.FillMissing{Value: qm.QuerySettings.FillValue, Mode: qm.QuerySettings.FillMode})
 		if err != nil {
@@ -231,7 +232,7 @@ func (d *SampleDatasource) CheckHealth(_ context.Context, req *backend.CheckHeal
 		}, nil
 	}
 
-	rows, err := ExecuteQueryx("SELECT 1")
+	rows, err := ExecuteQuery("SELECT 1")
 
 	if err != nil {
 		return &backend.CheckHealthResult{