Merge pull request #246 from jamf/feature/table_api

feature: Regatta server Tables API

cmd/common.go

@@ -157,6 +157,7 @@ func setupDragonboatLogger(logger *zap.Logger) {
 
 var secretConfigs = []string{
 	"maintenance.token",
+	"tables.token",
 }
 
 func viperConfigReader() map[string]any {

cmd/flags.go

@@ -19,6 +19,7 @@ var (
 	memberlistFlagSet   = pflag.NewFlagSet("memberlist", pflag.ContinueOnError)
 	storageFlagSet      = pflag.NewFlagSet("storage", pflag.ContinueOnError)
 	maintenanceFlagSet  = pflag.NewFlagSet("maintenance", pflag.ContinueOnError)
+	tablesFlagSet       = pflag.NewFlagSet("tables", pflag.ContinueOnError)
 	experimentalFlagSet = pflag.NewFlagSet("experimental", pflag.ContinueOnError)
 )
 
@@ -106,6 +107,10 @@ All nodes of the cluster MUST set this to the same value. If changing it is advi
 	// Maintenance flags
 	maintenanceFlagSet.Bool("maintenance.enabled", true, "Whether maintenance API is enabled.")
 	maintenanceFlagSet.String("maintenance.token", "", "Token to check for maintenance API access, if left empty (default) no token is checked.")
+
+	// Tables flags
+	tablesFlagSet.Bool("tables.enabled", true, "Whether tables API is enabled.")
+	tablesFlagSet.String("tables.token", "", "Token to check for tables API access, if left empty (default) no token is checked.")
 }
 
 func initConfig(set *pflag.FlagSet) {

cmd/follower.go

@@ -39,6 +39,7 @@ func init() {
 	followerCmd.PersistentFlags().AddFlagSet(memberlistFlagSet)
 	followerCmd.PersistentFlags().AddFlagSet(storageFlagSet)
 	followerCmd.PersistentFlags().AddFlagSet(maintenanceFlagSet)
+	followerCmd.PersistentFlags().AddFlagSet(tablesFlagSet)
 	followerCmd.PersistentFlags().AddFlagSet(experimentalFlagSet)
 
 	// Replication flags
@@ -203,6 +204,9 @@ func follower(_ *cobra.Command, _ []string) error {
 			if viper.GetBool("maintenance.enabled") {
 				regattapb.RegisterMaintenanceServer(regatta, &regattaserver.ResetServer{Tables: engine, AuthFunc: authFunc(viper.GetString("maintenance.token"))})
 			}
+			if viper.GetBool("tables.enabled") {
+				regattapb.RegisterTablesServer(regatta, &regattaserver.ReadonlyTablesServer{TablesServer: regattaserver.TablesServer{Tables: engine, AuthFunc: authFunc(viper.GetString("tables.token"))}})
+			}
 			// Start server
 			go func() {
 				if err := regatta.ListenAndServe(); err != nil {

cmd/leader.go

@@ -39,11 +39,14 @@ func init() {
 	leaderCmd.PersistentFlags().AddFlagSet(memberlistFlagSet)
 	leaderCmd.PersistentFlags().AddFlagSet(storageFlagSet)
 	leaderCmd.PersistentFlags().AddFlagSet(maintenanceFlagSet)
+	leaderCmd.PersistentFlags().AddFlagSet(tablesFlagSet)
 	leaderCmd.PersistentFlags().AddFlagSet(experimentalFlagSet)
 
 	// Tables flags
 	leaderCmd.PersistentFlags().StringSlice("tables.names", nil, "Create Regatta tables with given names.")
 	leaderCmd.PersistentFlags().StringSlice("tables.delete", nil, "Delete Regatta tables with given names.")
+	_ = leaderCmd.PersistentFlags().MarkHidden("tables.names")
+	_ = leaderCmd.PersistentFlags().MarkHidden("tables.delete")
 
 	// Replication flags
 	leaderCmd.PersistentFlags().Bool("replication.enabled", true, "Whether replication API is enabled.")
@@ -161,7 +164,7 @@ func leader(_ *cobra.Command, _ []string) error {
 		tNames := viper.GetStringSlice("tables.names")
 		for _, table := range tNames {
 			log.Debugf("creating table %s", table)
-			if err := engine.CreateTable(table); err != nil {
+			if _, err := engine.CreateTable(table); err != nil {
 				if errors.Is(err, serrors.ErrTableExists) {
 					log.Infof("table %s already exist, skipping creation", table)
 				} else {
@@ -194,6 +197,9 @@ func leader(_ *cobra.Command, _ []string) error {
 				Cluster: engine,
 				Config:  viperConfigReader,
 			})
+			if viper.GetBool("tables.enabled") {
+				regattapb.RegisterTablesServer(regatta, &regattaserver.TablesServer{Tables: engine, AuthFunc: authFunc(viper.GetString("tables.token"))})
+			}
 			if viper.GetBool("maintenance.enabled") {
 				regattapb.RegisterMaintenanceServer(regatta, &regattaserver.BackupServer{Tables: engine, AuthFunc: authFunc(viper.GetString("maintenance.token"))})
 			}
@@ -213,7 +219,7 @@ func leader(_ *cobra.Command, _ []string) error {
 				return fmt.Errorf("failed to create Replication server: %w", err)
 			}
 			ls := regattaserver.NewLogServer(
-				engine.Manager,
+				engine,
 				engine.LogReader,
 				logger,
 				viper.GetUint64("replication.max-send-message-size-bytes"),

docs/api.md

@@ -444,6 +444,57 @@ and generates events with the same revision for every completed request.
 It is allowed to modify the same key several times within one txn (the result will be the last Op that modified the key).
 
 
+# Tables {#regattav1tables}
+API for managing tables.
+## Create
+> **rpc** Create([CreateTableRequest](#createtablerequest))
+    [CreateTableResponse](#createtableresponse)
+
+Create a table. All followers will automatically replicate the table.
+This procedure is available only in the leader cluster.
+
+## Delete
+> **rpc** Delete([DeleteTableRequest](#deletetablerequest))
+    [DeleteTableResponse](#deletetableresponse)
+
+Delete a table. All followers will automatically delete the table.
+This procedure is available only in the leader cluster.
+
+## List
+> **rpc** List([ListTablesRequest](#listtablesrequest))
+    [ListTablesResponse](#listtablesresponse)
+
+Get names of all the tables present in the cluster.
+This procedure is available in both leader and follower clusters.
+
+
+
+
+
+<a name="regatta-v1-CreateTableRequest"></a>
+### CreateTableRequest
+CreateTableRequest describes the table to be created.
+
+| Field | Type | Label | Description |
+| ----- | ---- | ----- | ----------- |
+| name | [string](#string) |  | Name of the table to be created. |
+| config | [google.protobuf.Struct](#google-protobuf-Struct) |  | config the table configuration values. |
+
+
+
+
+
+
+<a name="regatta-v1-CreateTableResponse"></a>
+### CreateTableResponse
+CreateTableResponse describes the newly created table.
+
+| Field | Type | Label | Description |
+| ----- | ---- | ----- | ----------- |
+| id | [string](#string) |  | id the created table. |
+
+
+
 
 
 
@@ -479,6 +530,48 @@ It is allowed to modify the same key several times within one txn (the result wi
 
 
 
+<a name="regatta-v1-DeleteTableRequest"></a>
+### DeleteTableRequest
+DeleteTableRequest describes the table to be deleted.
+
+| Field | Type | Label | Description |
+| ----- | ---- | ----- | ----------- |
+| name | [string](#string) |  | name of the table to be deleted. |
+
+
+
+
+
+
+<a name="regatta-v1-DeleteTableResponse"></a>
+### DeleteTableResponse
+DeleteTableResponse when the table was successfully deleted.
+
+
+
+
+
+<a name="regatta-v1-ListTablesRequest"></a>
+### ListTablesRequest
+ListTablesRequest requests the list of currently registered tables.
+
+
+
+
+
+<a name="regatta-v1-ListTablesResponse"></a>
+### ListTablesResponse
+FollowerGetTablesResponse contains information about tables stored in the cluster.
+
+| Field | Type | Label | Description |
+| ----- | ---- | ----- | ----------- |
+| tables | [TableInfo](#regatta-v1-TableInfo) | repeated |  |
+
+
+
+
+
+
 <a name="regatta-v1-Member"></a>
 ### Member
 
@@ -648,6 +741,21 @@ It is allowed to modify the same key several times within one txn (the result wi
 
 
 
+<a name="regatta-v1-TableInfo"></a>
+### TableInfo
+TableInfo describes a single table.
+
+| Field | Type | Label | Description |
+| ----- | ---- | ----- | ----------- |
+| name | [string](#string) |  | name of the table. |
+| id | [string](#string) |  | id of the table. |
+| config | [google.protobuf.Struct](#google-protobuf-Struct) |  | config the table configuration values. |
+
+
+
+
+
+
 <a name="regatta-v1-TableStatus"></a>
 ### TableStatus
 

docs/operations_guide/cli/regatta_follower.md

@@ -88,6 +88,8 @@ regatta follower [flags]
       --rest.read-timeout duration                            Maximum duration for reading the entire request. (default 5s)
       --storage.block-cache-size int                          Shared block cache size in bytes, the cache is used to hold uncompressed blocks of data in memory. (default 16777216)
       --storage.table-cache-size int                          Shared table cache size, the cache is used to hold handles to open SSTs. (default 1024)
+      --tables.enabled                                        Whether tables API is enabled. (default true)
+      --tables.token string                                   Token to check for tables API access, if left empty (default) no token is checked.
 ```
 
 ### SEE ALSO

docs/operations_guide/cli/regatta_leader.md

@@ -82,8 +82,8 @@ regatta leader [flags]
       --rest.read-timeout duration                     Maximum duration for reading the entire request. (default 5s)
       --storage.block-cache-size int                   Shared block cache size in bytes, the cache is used to hold uncompressed blocks of data in memory. (default 16777216)
       --storage.table-cache-size int                   Shared table cache size, the cache is used to hold handles to open SSTs. (default 1024)
-      --tables.delete strings                          Delete Regatta tables with given names.
-      --tables.names strings                           Create Regatta tables with given names.
+      --tables.enabled                                 Whether tables API is enabled. (default true)
+      --tables.token string                            Token to check for tables API access, if left empty (default) no token is checked.
 ```
 
 ### SEE ALSO

proto/regatta.proto

@@ -197,7 +197,6 @@ service Cluster {
 
   // Status gets the status of the member.
   rpc Status(StatusRequest) returns (StatusResponse);
-
 }
 
 message MemberListRequest {}
@@ -254,3 +253,62 @@ message StatusResponse {
   // errors contains alarm/health information and status.
   repeated string errors = 8;
 }
+
+
+// API for managing tables.
+service Tables {
+  // Create a table. All followers will automatically replicate the table.
+  // This procedure is available only in the leader cluster.
+  rpc Create(CreateTableRequest) returns (CreateTableResponse);
+
+  // Delete a table. All followers will automatically delete the table.
+  // This procedure is available only in the leader cluster.
+  rpc Delete(DeleteTableRequest) returns (DeleteTableResponse);
+
+  // Get names of all the tables present in the cluster.
+  // This procedure is available in both leader and follower clusters.
+  rpc List(ListTablesRequest) returns (ListTablesResponse);
+}
+
+// CreateTableRequest describes the table to be created.
+message CreateTableRequest {
+  // Name of the table to be created.
+  string name = 1;
+  // config the table configuration values.
+  google.protobuf.Struct config = 5;
+}
+
+// CreateTableResponse describes the newly created table.
+message CreateTableResponse {
+  // id the created table.
+  string id = 1;
+}
+
+// DeleteTableRequest describes the table to be deleted.
+message DeleteTableRequest {
+  // name of the table to be deleted.
+  string name = 1;
+}
+
+// DeleteTableResponse when the table was successfully deleted.
+message DeleteTableResponse {}
+
+// ListTablesRequest requests the list of currently registered tables.
+message ListTablesRequest {}
+
+// TableInfo describes a single table.
+message TableInfo {
+  // name of the table.
+  string name = 1;
+
+  // id of the table.
+  string id = 2;
+
+  // config the table configuration values.
+  google.protobuf.Struct config = 5;
+}
+
+// FollowerGetTablesResponse contains information about tables stored in the cluster.
+message ListTablesResponse {
+  repeated TableInfo tables = 1;
+}