Skip to content

JiT context RPC service and protos #2079

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ikholopov-omni merged 1 commit into from
Feb 4, 2026
+215 −86
Merged

JiT context RPC service and protos #2079

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions packages/@dataform/core/BUILD
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,18 @@ add_license_header_to_file(
to_file = "configs.proto",
)

add_license_header_to_file(
name = "db_adapter_proto_with_license",
from_file = "//protos:db_adapter.proto",
to_file = "db_adapter.proto",
)

add_license_header_to_file(
name = "jit_proto_with_license",
from_file = "//protos:jit.proto",
to_file = "jit.proto",
)

add_license_header_to_file(
name = "bundle_with_license",
from_file = "bundle",
Expand Down Expand Up @@ -95,6 +107,8 @@ pkg_npm_tar(
":bundle_with_license",
":configs_proto_with_license",
":core_proto_with_license",
":db_adapter_proto_with_license",
":jit_proto_with_license",
":package.json",
"//:version.bzl",
],
Expand Down
8 changes: 6 additions & 2 deletions protos/BUILD
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
load("@rules_proto//proto:defs.bzl", "proto_library")
load("//testing:build_test.bzl", "build_test")
load("//tools:ts_proto_library.bzl", "ts_proto_library")

package(default_visibility = ["//visibility:public"])
Expand All @@ -9,11 +8,16 @@ proto_library(
srcs = [
"configs.proto",
"core.proto",
"db_adapter.proto",
"evaluation.proto",
"execution.proto",
"jit.proto",
"profiles.proto",
],
deps = ["@com_google_protobuf//:struct_proto"],
deps = [
"@com_google_protobuf//:empty_proto",
"@com_google_protobuf//:struct_proto",
],
)

ts_proto_library(
Expand Down
78 changes: 78 additions & 0 deletions protos/db_adapter.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
syntax = "proto3";

package dataform;

import "protos/core.proto";

// Common structures for Evaluation, execution and compilation.

message Field {
enum Primitive {
UNKNOWN = 0;
INTEGER = 1;
FLOAT = 2;
NUMERIC = 5;
BOOLEAN = 3;
STRING = 4;
DATE = 6;
DATETIME = 7;
TIMESTAMP = 8;
TIME = 9;
BYTES = 10;
ANY = 11;
GEOGRAPHY = 12;
}

enum Flag {
UNKNOWN_FLAG = 0;
REPEATED = 1;
}

string name = 1;

repeated Flag flags = 6;

oneof type {
Primitive primitive = 7;
Fields struct = 3;
}

string description = 5;

reserved 2, 4;
}

message Fields {
repeated Field fields = 1;
}

message TableMetadata {
Target target = 1;
enum Type {
UNKNOWN = 0;
TABLE = 1;
VIEW = 2;
}
Type type = 6;

repeated Field fields = 3;
string description = 5;
map<string, string> labels = 7;
int64 last_updated_millis = 4;

message BigQuery {
bool has_streaming_buffer = 1;
}
BigQuery bigquery = 8;

reserved 2;
}

message ExecutionMetadata {
message BigqueryMetadata {
string job_id = 1;
int64 total_bytes_processed = 2;
int64 total_bytes_billed = 3;
}
BigqueryMetadata bigquery = 1;
}
93 changes: 9 additions & 84 deletions protos/execution.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,20 +1,20 @@
syntax="proto3";
syntax = "proto3";

package dataform;

import "protos/core.proto";
import "protos/db_adapter.proto";

option go_package = "github.com/dataform-co/dataform/protos/dataform";


message RunConfig {
repeated string actions = 1;
repeated string tags = 5;
bool include_dependencies = 3;
bool include_dependents = 11;
bool full_refresh = 2;
int32 timeout_millis = 7;

// For internal use only, will be removed at a later date.
bool disable_set_metadata = 9;

Expand All @@ -36,7 +36,6 @@ message ExecutionAction {
string type = 4;
string table_type = 6;


repeated Target dependency_targets = 11;
ActionHermeticity hermeticity = 10;

Expand Down Expand Up @@ -82,7 +81,6 @@ message RunResult {
}

message ActionResult {

Target target = 5;

enum ExecutionStatus {
Expand All @@ -103,24 +101,14 @@ message ActionResult {
reserved 1, 6, 7;
}


message ExecutionMetadata {
message BigqueryMetadata {
string job_id = 1;
int64 total_bytes_processed = 2;
int64 total_bytes_billed = 3;
}
BigqueryMetadata bigquery = 1;
}

message TaskResult {
enum ExecutionStatus {
UNKNOWN = 0;
RUNNING = 1;
SUCCESSFUL = 2;
FAILED = 3;
SKIPPED = 4;
CANCELLED = 5;
UNKNOWN = 0;
RUNNING = 1;
SUCCESSFUL = 2;
FAILED = 3;
SKIPPED = 4;
CANCELLED = 5;
}
ExecutionStatus status = 1;
string error_message = 2;
Expand All @@ -133,66 +121,3 @@ message TestResult {
bool successful = 2;
repeated string messages = 3;
}

message Field {

enum Primitive {
UNKNOWN = 0;
INTEGER = 1;
FLOAT = 2;
NUMERIC = 5;
BOOLEAN = 3;
STRING = 4;
DATE = 6;
DATETIME = 7;
TIMESTAMP = 8;
TIME = 9;
BYTES = 10;
ANY = 11;
GEOGRAPHY = 12;
}

enum Flag {
UNKNOWN_FLAG = 0;
REPEATED = 1;
}

string name = 1;

repeated Flag flags = 6;

oneof type {
Primitive primitive = 7;
Fields struct = 3;
}

string description = 5;

reserved 2, 4;
}

message Fields {
repeated Field fields = 1;
}

message TableMetadata {
Target target = 1;
enum Type {
UNKNOWN = 0;
TABLE = 1;
VIEW = 2;
}
Type type = 6;

repeated Field fields = 3;
string description = 5;
map<string, string> labels = 7;
int64 last_updated_millis = 4;

message BigQuery {
bool has_streaming_buffer = 1;
}
BigQuery bigquery = 8;

reserved 2;
}
108 changes: 108 additions & 0 deletions protos/jit.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
syntax = "proto3";

package dataform;

import "google/protobuf/empty.proto";
import "google/protobuf/struct.proto";
import "protos/core.proto";
import "protos/db_adapter.proto";

// Database adapter, available in JiT compilation context.
service DbAdapter {
// Executes a query.
rpc Execute(ExecuteRequest) returns (ExecuteResponse) {}

// Lists table in schema.
rpc ListTables(ListTablesRequest) returns (ListTablesResponse) {}

// Gets table for target.
rpc GetTable(GetTableRequest) returns (TableMetadata) {}

// Drops table of specified target.
rpc DeleteTable(DeleteTableRequest) returns (google.protobuf.Empty) {}
}

// Request to execute synchronous SQL query.
message ExecuteRequest {
// SQL statement query.
string statement = 1;
// Max rows to return in the job.
int64 row_limit = 2;
// Max bytes to process.
int64 byte_limit = 3;
// Whether or not fetch result rows.
bool fetch_results = 4;

BigQueryExecuteOptions big_query_options = 5;
}

message BigQueryExecuteOptions {
// Priority - interactive if true or batch if false (default).
bool interactive = 1;
// See https://docs.cloud.google.com/bigquery/docs/reference/legacy-sql.
bool use_legacy_sql = 2;
// Labels to add to the job.
map<string, string> labels = 3;
// Location to execute the job at.
string location = 4;
// Job prefix to add.
string job_prefix = 5;
// Is dry run job.
bool dry_run = 6;
}

// Synchronous execution response result.
message ExecuteResponse {
// Job metadata.
ExecutionMetadata execution_metadata = 1;
// Rows. For BigQuery, see
// https://docs.cloud.google.com/bigquery/docs/reference/rest/v2/jobs/getQueryResults.
repeated google.protobuf.Struct rows = 2;
repeated string errors = 3;
}

// Request to list tables in schema.
message ListTablesRequest {
string database = 1;
string schema = 2;
}

// Tables metadata in schema.
message ListTablesResponse {
repeated TableMetadata tables = 1;
}

// Request to get table metadata for target.
message GetTableRequest {
Target target = 1;
}

// Request to drop the table for target.
message DeleteTableRequest {
Target target = 1;
}

// JiT compilation result for table actions (including views).
// Fields match the Table message.
message JitTableResult {
// SQL Select query of the table.
string query = 1;
// Optional pre-operation statements.
repeated string pre_ops = 3;
// Optional post-operation statements.
repeated string post_ops = 4;
}

// JiT compilation result for incremental table actions.
message JitIncrementalTableResult {
// Fields match the Table message.
JitTableResult regular = 1;
// Fields match incremental_ fields in the Table message.
JitTableResult incremental = 2;
}

// JiT compilation result for operation actions.
message JitOperationResult {
// Sequence of SQL operations.
repeated string queries = 1;
}
Loading