Merge pull request #192 from stackql/feature/stackql-typing

stackql-typing

docs/type_system.md

@@ -0,0 +1,29 @@
+
+# __`stackql`__ type system
+
+There exists a relation from openapi document (discovery document) type and relational type, we dub it "discovery-relational mapping (DRM)":
+
+$\ R_{drm}: \text{discovery-type} \to \text{relational-type}\ \ \ \ \ (1) $
+
+In addition, the "traditional" object-relational mapping relation exists:
+
+$\ Q_{orm}: \text{relational-type} \to \text{golang-type}\ \ \ \ \ (2) $
+
+These relations are mapped out in:
+
+- [internal/stackql/drm](/internal/stackql/drm).
+- [internal/stackql/typing](/internal/stackql/typing).
+
+The `golang` `sql` driver is used: 
+
+```go
+import (
+    "database/sql"
+)
+
+
+var (
+    _ *sql.ColumnType = (*sql.ColumnType)(nil) // This is the golang SQL driver type
+
+)
+```

internal/stackql/astindirect/indirect.go

@@ -6,6 +6,7 @@ import (
 	"github.com/stackql/go-openapistackql/openapistackql"
 	"github.com/stackql/stackql/internal/stackql/drm"
 	"github.com/stackql/stackql/internal/stackql/symtab"
+	"github.com/stackql/stackql/internal/stackql/typing"
 
 	"github.com/stackql/stackql-parser/go/vt/sqlparser"
 	"github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto"
@@ -48,8 +49,8 @@ func NewSubqueryIndirect(subQueryDTO internaldto.SubqueryDTO) (Indirect, error)
 type Indirect interface {
 	Parse() error
 	GetAssignedParameters() (internaldto.TableParameterCollection, bool)
-	GetColumnByName(name string) (internaldto.ColumnMetadata, bool)
-	GetColumns() []internaldto.ColumnMetadata
+	GetColumnByName(name string) (typing.ColumnMetadata, bool)
+	GetColumns() []typing.ColumnMetadata
 	GetName() string
 	GetOptionalParameters() map[string]openapistackql.Addressable
 	GetRequiredParameters() map[string]openapistackql.Addressable

internal/stackql/astindirect/subquery_indirect.go

@@ -6,6 +6,7 @@ import (
 	"github.com/stackql/stackql/internal/stackql/drm"
 	"github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto"
 	"github.com/stackql/stackql/internal/stackql/symtab"
+	"github.com/stackql/stackql/internal/stackql/typing"
 )
 
 type subquery struct {
@@ -41,7 +42,7 @@ func (v *subquery) GetName() string {
 	return v.subQueryDTO.GetAlias().GetRawVal()
 }
 
-func (v *subquery) GetColumns() []internaldto.ColumnMetadata {
+func (v *subquery) GetColumns() []typing.ColumnMetadata {
 	return v.selCtx.GetNonControlColumns()
 }
 
@@ -53,7 +54,7 @@ func (v *subquery) GetRequiredParameters() map[string]openapistackql.Addressable
 	return nil
 }
 
-func (v *subquery) GetColumnByName(name string) (internaldto.ColumnMetadata, bool) {
+func (v *subquery) GetColumnByName(name string) (typing.ColumnMetadata, bool) {
 	for _, col := range v.selCtx.GetNonControlColumns() {
 		if col.GetIdentifier() == name {
 			return col, true

internal/stackql/astindirect/view_indirect.go

@@ -9,6 +9,7 @@ import (
 	"github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto"
 	"github.com/stackql/stackql/internal/stackql/parser"
 	"github.com/stackql/stackql/internal/stackql/symtab"
+	"github.com/stackql/stackql/internal/stackql/typing"
 )
 
 type view struct {
@@ -43,7 +44,7 @@ func (v *view) GetName() string {
 	return v.viewDTO.GetName()
 }
 
-func (v *view) GetColumns() []internaldto.ColumnMetadata {
+func (v *view) GetColumns() []typing.ColumnMetadata {
 	return v.selCtx.GetNonControlColumns()
 }
 
@@ -55,7 +56,7 @@ func (v *view) GetRequiredParameters() map[string]openapistackql.Addressable {
 	return nil
 }
 
-func (v *view) GetColumnByName(name string) (internaldto.ColumnMetadata, bool) {
+func (v *view) GetColumnByName(name string) (typing.ColumnMetadata, bool) {
 	nccs := v.selCtx.GetNonControlColumns()
 	for _, col := range nccs {
 		if col.GetIdentifier() == name {

internal/stackql/bundle/bundle.go

@@ -12,6 +12,7 @@ import (
 	"github.com/stackql/stackql/internal/stackql/sqlcontrol"
 	"github.com/stackql/stackql/internal/stackql/sqlengine"
 	"github.com/stackql/stackql/internal/stackql/tablenamespace"
+	"github.com/stackql/stackql/internal/stackql/typing"
 	"github.com/stackql/stackql/pkg/txncounter"
 )
 
@@ -41,6 +42,7 @@ func NewBundle(
 	authContexts map[string]*dto.AuthCtx,
 	sqlDataSources map[string]sql_datasource.SQLDataSource,
 	txnCoordintatorContext txn_context.ITransactionCoordinatorContext,
+	typCfg typing.Config,
 ) Bundle {
 	return &simpleBundle{
 		garbageCollector:       garbageCollector,
@@ -55,6 +57,7 @@ func NewBundle(
 		authContexts:           authContexts,
 		sqlDataSources:         sqlDataSources,
 		txnCoordintatorContext: txnCoordintatorContext,
+		typCfg:                 typCfg,
 	}
 }
 
@@ -66,6 +69,7 @@ type simpleBundle struct {
 	sqlSystem              sql_system.SQLSystem
 	txnStore               kstore.KStore
 	txnCtrMgr              txncounter.Manager
+	typCfg                 typing.Config
 	formatter              sqlparser.NodeFormatter
 	pgInternalRouter       dbmsinternal.Router
 	sqlDataSources         map[string]sql_datasource.SQLDataSource
@@ -105,6 +109,10 @@ func (sb *simpleBundle) GetTxnCounterManager() txncounter.Manager {
 	return sb.txnCtrMgr
 }
 
+func (sb *simpleBundle) GetTypingConfig() typing.Config {
+	return sb.typCfg
+}
+
 func (sb *simpleBundle) GetGC() garbagecollector.GarbageCollector {
 	return sb.garbageCollector
 }

internal/stackql/data_staging/output_data_staging/output_data_staging.go

@@ -14,6 +14,7 @@ import (
 	"github.com/stackql/stackql/internal/stackql/logging"
 	"github.com/stackql/stackql/internal/stackql/sqlmachinery"
 	"github.com/stackql/stackql/internal/stackql/streaming"
+	"github.com/stackql/stackql/internal/stackql/typing"
 	"github.com/stackql/stackql/internal/stackql/util"
 )
 
@@ -53,7 +54,7 @@ type PacketPreparator interface {
 
 func NewNaivePacketPreparator(
 	source Source,
-	nonControlColumns []internaldto.ColumnMetadata,
+	nonControlColumns []typing.ColumnMetadata,
 	stream streaming.MapStream,
 	drmCfg drm.Config,
 ) PacketPreparator {
@@ -67,7 +68,7 @@ func NewNaivePacketPreparator(
 
 type naivePacketPreparator struct {
 	source            Source
-	nonControlColumns []internaldto.ColumnMetadata
+	nonControlColumns []typing.ColumnMetadata
 	stream            streaming.MapStream
 	drmCfg            drm.Config
 }
@@ -98,7 +99,7 @@ type Outputter interface {
 	OutputExecutorResult() internaldto.ExecutorOutput
 }
 
-func NewNaiveOutputter(packetPreparator PacketPreparator, nonControlColumns []internaldto.ColumnMetadata) Outputter {
+func NewNaiveOutputter(packetPreparator PacketPreparator, nonControlColumns []typing.ColumnMetadata) Outputter {
 	return &naiveOutputter{
 		packetPreparator:  packetPreparator,
 		nonControlColumns: nonControlColumns,
@@ -107,7 +108,7 @@ func NewNaiveOutputter(packetPreparator PacketPreparator, nonControlColumns []in
 
 type naiveOutputter struct {
 	packetPreparator  PacketPreparator
-	nonControlColumns []internaldto.ColumnMetadata
+	nonControlColumns []typing.ColumnMetadata
 }
 
 func (st *naiveOutputter) OutputExecutorResult() internaldto.ExecutorOutput {

internal/stackql/drm/drm_cfg.go

@@ -9,7 +9,6 @@ import (
 	"strings"
 
 	"github.com/stackql/stackql/internal/stackql/constants"
-	"github.com/stackql/stackql/internal/stackql/internal_data_transfer/internal_relational_dto"
 	"github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto"
 	"github.com/stackql/stackql/internal/stackql/internal_data_transfer/relationaldto"
 	"github.com/stackql/stackql/internal/stackql/logging"
@@ -19,6 +18,7 @@ import (
 	"github.com/stackql/stackql/internal/stackql/sqlmachinery"
 	"github.com/stackql/stackql/internal/stackql/streaming"
 	"github.com/stackql/stackql/internal/stackql/tablenamespace"
+	"github.com/stackql/stackql/internal/stackql/typing"
 	"github.com/stackql/stackql/internal/stackql/util"
 
 	"github.com/stackql/go-openapistackql/openapistackql"
@@ -35,20 +35,20 @@ var (
 )
 
 type Config interface {
-	ColumnsToRelationalColumns(cols []internaldto.ColumnMetadata) []relationaldto.RelationalColumn
-	ColumnToRelationalColumn(cols internaldto.ColumnMetadata) relationaldto.RelationalColumn
+	ColumnsToRelationalColumns(cols []typing.ColumnMetadata) []relationaldto.RelationalColumn
+	ColumnToRelationalColumn(cols typing.ColumnMetadata) relationaldto.RelationalColumn
 	ExtractFromGolangValue(interface{}) interface{}
 	ExtractObjectFromSQLRows(
 		r *sql.Rows,
-		nonControlColumns []internaldto.ColumnMetadata,
+		nonControlColumns []typing.ColumnMetadata,
 		stream streaming.MapStream,
 	) (map[string]map[string]interface{}, map[int]map[int]interface{})
 	GetCurrentTable(internaldto.HeirarchyIdentifiers) (internaldto.DBTable, error)
 	GetRelationalType(string) string
 	GenerateDDL(util.AnnotatedTabulation, openapistackql.OperationStore, int, bool) ([]string, error)
 	GetControlAttributes() sqlcontrol.ControlAttributes
 	GetGolangValue(string) interface{}
-	GetGolangSlices([]internaldto.ColumnMetadata) ([]interface{}, []string)
+	GetGolangSlices([]typing.ColumnMetadata) ([]interface{}, []string)
 	GetNamespaceCollection() tablenamespace.Collection
 	GetParserTableName(internaldto.HeirarchyIdentifiers, int) sqlparser.TableName
 	GetSQLSystem() sql_system.SQLSystem
@@ -112,7 +112,7 @@ func (dc *staticDRMConfig) OpenapiColumnsToRelationalColumns(
 			col.GetQualifier(),
 		).WithAlias(col.GetAlias()).WithDecorated(col.GetDecoratedCol()).WithParserNode(col.GetNode())
 		if schemaExists {
-			inferredOID := internaldto.GetOidForSchema(col.GetSchema())
+			inferredOID := typing.GetOidForSchema(col.GetSchema())
 			relationalColumn = relationalColumn.WithOID(inferredOID)
 		}
 		// TODO: Need a way to handle postgres differences. This is a fragile point
@@ -153,7 +153,7 @@ func (dc *staticDRMConfig) OpenapiColumnsToRelationalColumn(
 		typeStr,
 	).WithQualifier(col.GetQualifier()).WithAlias(col.GetAlias()).WithDecorated(decoratedCol).WithParserNode(col.GetNode())
 	if schemaExists {
-		inferredOID := internaldto.GetOidForSchema(col.GetSchema())
+		inferredOID := typing.GetOidForSchema(col.GetSchema())
 		relationalColumn = relationalColumn.WithOID(inferredOID)
 	}
 	// TODO: Need a way to handle postgres differences
@@ -162,7 +162,7 @@ func (dc *staticDRMConfig) OpenapiColumnsToRelationalColumn(
 }
 
 func (dc *staticDRMConfig) ColumnsToRelationalColumns(
-	cols []internaldto.ColumnMetadata,
+	cols []typing.ColumnMetadata,
 ) []relationaldto.RelationalColumn {
 	var relationalColumns []relationaldto.RelationalColumn
 	for _, col := range cols {
@@ -174,7 +174,7 @@ func (dc *staticDRMConfig) ColumnsToRelationalColumns(
 }
 
 func (dc *staticDRMConfig) ColumnToRelationalColumn(
-	col internaldto.ColumnMetadata,
+	col typing.ColumnMetadata,
 ) relationaldto.RelationalColumn {
 	relationalColumn := relationaldto.NewRelationalColumn(
 		col.GetName(),
@@ -191,21 +191,21 @@ func (dc *staticDRMConfig) getControlAttributes() sqlcontrol.ControlAttributes {
 	return dc.controlAttributes
 }
 
-func (dc *staticDRMConfig) GetGolangSlices(nonControlColumns []internaldto.ColumnMetadata) ([]interface{}, []string) {
+func (dc *staticDRMConfig) GetGolangSlices(nonControlColumns []typing.ColumnMetadata) ([]interface{}, []string) {
 	return dc.getGolangSlices(nonControlColumns)
 }
 
 func (dc *staticDRMConfig) ExtractObjectFromSQLRows(
 	r *sql.Rows,
-	nonControlColumns []internaldto.ColumnMetadata,
+	nonControlColumns []typing.ColumnMetadata,
 	stream streaming.MapStream,
 ) (map[string]map[string]interface{}, map[int]map[int]interface{}) {
 	return dc.extractObjectFromSQLRows(r, nonControlColumns, stream)
 }
 
 func (dc *staticDRMConfig) extractObjectFromSQLRows(
 	r *sql.Rows,
-	nonControlColumns []internaldto.ColumnMetadata,
+	nonControlColumns []typing.ColumnMetadata,
 	stream streaming.MapStream,
 ) (map[string]map[string]interface{}, map[int]map[int]interface{}) {
 	if r != nil {
@@ -256,7 +256,7 @@ func (dc *staticDRMConfig) extractObjectFromSQLRows(
 	return altKeys, rawRows
 }
 
-func (dc *staticDRMConfig) getGolangSlices(nonControlColumns []internaldto.ColumnMetadata) ([]interface{}, []string) {
+func (dc *staticDRMConfig) getGolangSlices(nonControlColumns []typing.ColumnMetadata) ([]interface{}, []string) {
 	i := 0
 	var keyArr []string
 	var ifArr []interface{}
@@ -459,7 +459,7 @@ func (dc *staticDRMConfig) GenerateInsertDML(
 	method openapistackql.OperationStore,
 	tcc internaldto.TxnControlCounters,
 ) (PreparedStatementCtx, error) {
-	var columns []internaldto.ColumnMetadata
+	var columns []typing.ColumnMetadata
 	_, isSQLDataSource := tabAnnotated.GetSQLDataSource()
 	var tableName string
 	var discoverID int
@@ -497,7 +497,7 @@ func (dc *staticDRMConfig) GenerateInsertDML(
 			return nil, err
 		}
 		for _, col := range tableColumns {
-			columns = append(columns, internal_relational_dto.NewRelayedColDescriptor(col, col.GetType()))
+			columns = append(columns, typing.NewRelayedColDescriptor(col, col.GetType()))
 			relationalTable.PushBackColumn(col)
 		}
 	} else {
@@ -512,7 +512,7 @@ func (dc *staticDRMConfig) GenerateInsertDML(
 			if schema != nil && schema.GetType() != "" {
 				relationalType = dc.GetRelationalType(schema.GetType())
 			}
-			columns = append(columns, internaldto.NewColDescriptor(col, relationalType))
+			columns = append(columns, typing.NewColDescriptor(col, relationalType))
 			relationalColumn := relationaldto.NewRelationalColumn(col.GetName(), relationalType).WithParserNode(col.GetNode())
 			relationalTable.PushBackColumn(relationalColumn)
 		}
@@ -547,7 +547,7 @@ func (dc *staticDRMConfig) GenerateSelectDML(
 	rewrittenWhere string,
 ) (PreparedStatementCtx, error) {
 	var quotedColNames []string
-	var columns []internaldto.ColumnMetadata
+	var columns []typing.ColumnMetadata
 
 	aliasStr := ""
 	if tabAnnotated.GetAlias() != "" {
@@ -575,7 +575,7 @@ func (dc *staticDRMConfig) GenerateSelectDML(
 				}
 			}
 		}
-		columns = append(columns, internaldto.NewColDescriptor(col, typeStr))
+		columns = append(columns, typing.NewColDescriptor(col, typeStr))
 		// TODO: logic to infer column width
 		relationalColumn := relationaldto.NewRelationalColumn(
 			col.GetName(),

internal/stackql/drm/prepared_statement_ctx.go

@@ -6,14 +6,15 @@ import (
 	"github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto"
 	"github.com/stackql/stackql/internal/stackql/sql_system"
 	"github.com/stackql/stackql/internal/stackql/tablenamespace"
+	"github.com/stackql/stackql/internal/stackql/typing"
 )
 
 type PreparedStatementCtx interface {
 	GetAllCtrlCtrs() []internaldto.TxnControlCounters
 	GetGCCtrlCtrs() internaldto.TxnControlCounters
 	GetIndirectContexts() []PreparedStatementCtx
 	GetCtrlColumnRepeats() int
-	GetNonControlColumns() []internaldto.ColumnMetadata
+	GetNonControlColumns() []typing.ColumnMetadata
 	GetGCHousekeepingQueries() string
 	GetQuery() string
 	SetGCCtrlCtrs(tcc internaldto.TxnControlCounters)
@@ -30,7 +31,7 @@ type standardPreparedStatementCtx struct {
 	txnIDControlColName     string
 	insIDControlColName     string
 	insEncodedColName       string
-	nonControlColumns       []internaldto.ColumnMetadata
+	nonControlColumns       []typing.ColumnMetadata
 	ctrlColumnRepeats       int
 	txnCtrlCtrs             internaldto.TxnControlCounters
 	selectTxnCtrlCtrs       []internaldto.TxnControlCounters
@@ -67,7 +68,7 @@ func (ps *standardPreparedStatementCtx) SetGCCtrlCtrs(tcc internaldto.TxnControl
 	ps.txnCtrlCtrs = tcc
 }
 
-func (ps *standardPreparedStatementCtx) GetNonControlColumns() []internaldto.ColumnMetadata {
+func (ps *standardPreparedStatementCtx) GetNonControlColumns() []typing.ColumnMetadata {
 	return ps.nonControlColumns
 }
 
@@ -87,7 +88,7 @@ func NewPreparedStatementCtx(
 	txnIDControlColName string,
 	insIDControlColName string,
 	insEncodedColName string,
-	nonControlColumns []internaldto.ColumnMetadata,
+	nonControlColumns []typing.ColumnMetadata,
 	ctrlColumnRepeats int,
 	txnCtrlCtrs internaldto.TxnControlCounters,
 	secondaryCtrs []internaldto.TxnControlCounters,
@@ -112,7 +113,7 @@ func NewPreparedStatementCtx(
 	}
 }
 
-func NewQueryOnlyPreparedStatementCtx(query string, nonControlCols []internaldto.ColumnMetadata) PreparedStatementCtx {
+func NewQueryOnlyPreparedStatementCtx(query string, nonControlCols []typing.ColumnMetadata) PreparedStatementCtx {
 	return &standardPreparedStatementCtx{query: query, nonControlColumns: nonControlCols, ctrlColumnRepeats: 0}
 }