From 7a5690a597c4fcd411d731a27f32017b41037a05 Mon Sep 17 00:00:00 2001 From: Benjamin Yolken Date: Mon, 23 Nov 2020 22:35:43 -0800 Subject: [PATCH 01/19] Start working on connector refactoring --- cmd/topicctl/subcmd/get.go | 6 ++- cmd/topicctl/subcmd/repl.go | 6 ++- cmd/topicctl/subcmd/reset.go | 6 ++- cmd/topicctl/subcmd/tail.go | 6 ++- pkg/admin/brokerclient.go | 32 +++++++------- pkg/admin/brokerclient_test.go | 42 +++++++++++++++--- pkg/admin/connector.go | 68 +++++++++++++++++++++++++++++ pkg/cli/cli.go | 25 +++++++---- pkg/cli/repl.go | 13 +++++- pkg/config/cluster.go | 6 ++- pkg/groups/client_test.go | 26 +++++++++-- pkg/groups/{client.go => groups.go} | 42 +++++++----------- pkg/messages/bounds.go | 23 ++++++---- 13 files changed, 220 insertions(+), 81 deletions(-) create mode 100644 pkg/admin/connector.go rename pkg/groups/{client.go => groups.go} (82%) diff --git a/cmd/topicctl/subcmd/get.go b/cmd/topicctl/subcmd/get.go index cd901de5..4acaccbb 100644 --- a/cmd/topicctl/subcmd/get.go +++ b/cmd/topicctl/subcmd/get.go @@ -107,8 +107,10 @@ func getRun(cmd *cobra.Command, args []string) error { adminClient, clientErr = admin.NewBrokerAdminClient( ctx, admin.BrokerAdminClientConfig{ - BrokerAddr: getConfig.brokerAddr, - ReadOnly: true, + BrokerClientConfig: admin.BrokerClientConfig{ + BrokerAddr: getConfig.brokerAddr, + }, + ReadOnly: true, }, ) } else { diff --git a/cmd/topicctl/subcmd/repl.go b/cmd/topicctl/subcmd/repl.go index e8b47c03..17eca542 100644 --- a/cmd/topicctl/subcmd/repl.go +++ b/cmd/topicctl/subcmd/repl.go @@ -89,8 +89,10 @@ func replRun(cmd *cobra.Command, args []string) error { adminClient, clientErr = admin.NewBrokerAdminClient( ctx, admin.BrokerAdminClientConfig{ - BrokerAddr: replConfig.brokerAddr, - ReadOnly: true, + BrokerClientConfig: admin.BrokerClientConfig{ + BrokerAddr: replConfig.brokerAddr, + }, + ReadOnly: true, }, ) } else { diff --git a/cmd/topicctl/subcmd/reset.go b/cmd/topicctl/subcmd/reset.go index aa697d56..27878771 100644 --- a/cmd/topicctl/subcmd/reset.go +++ b/cmd/topicctl/subcmd/reset.go @@ -107,8 +107,10 @@ func resetOffsetsRun(cmd *cobra.Command, args []string) error { adminClient, clientErr = admin.NewBrokerAdminClient( ctx, admin.BrokerAdminClientConfig{ - BrokerAddr: resetOffsetsConfig.brokerAddr, - ReadOnly: true, + BrokerClientConfig: admin.BrokerClientConfig{ + BrokerAddr: resetOffsetsConfig.brokerAddr, + }, + ReadOnly: true, }, ) } else { diff --git a/cmd/topicctl/subcmd/tail.go b/cmd/topicctl/subcmd/tail.go index 88481d88..59a47b8d 100644 --- a/cmd/topicctl/subcmd/tail.go +++ b/cmd/topicctl/subcmd/tail.go @@ -127,8 +127,10 @@ func tailRun(cmd *cobra.Command, args []string) error { adminClient, clientErr = admin.NewBrokerAdminClient( ctx, admin.BrokerAdminClientConfig{ - BrokerAddr: tailConfig.brokerAddr, - ReadOnly: true, + BrokerClientConfig: admin.BrokerClientConfig{ + BrokerAddr: tailConfig.brokerAddr, + }, + ReadOnly: true, }, ) } else { diff --git a/pkg/admin/brokerclient.go b/pkg/admin/brokerclient.go index 081ffbd1..ff11462b 100644 --- a/pkg/admin/brokerclient.go +++ b/pkg/admin/brokerclient.go @@ -19,26 +19,28 @@ const ( // BrokerAdminClient is a Client implementation that only uses broker APIs, without any // zookeeper access. type BrokerAdminClient struct { - brokerAddr string client *kafka.Client - readOnly bool + connector *BrokerConnector + config BrokerAdminClientConfig supportedFeatures SupportedFeatures } var _ Client = (*BrokerAdminClient)(nil) type BrokerAdminClientConfig struct { - BrokerAddr string - ReadOnly bool + BrokerConnectorConfig + ReadOnly bool } func NewBrokerAdminClient( ctx context.Context, config BrokerAdminClientConfig, ) (*BrokerAdminClient, error) { - client := &kafka.Client{ - Addr: kafka.TCP(config.BrokerAddr), + brokerConnector, err := NewBrokerConnector(config.BrokerConnectorConfig) + if err != nil { + return nil, err } + client := brokerConnector.KafkaClient apiVersions, err := client.ApiVersions(ctx, kafka.ApiVersionsRequest{}) if err != nil { @@ -80,9 +82,9 @@ func NewBrokerAdminClient( log.Debugf("Supported features: %+v", supportedFeatures) return &BrokerAdminClient{ - brokerAddr: config.BrokerAddr, client: client, - readOnly: config.ReadOnly, + connector: brokerConnector, + config: config, supportedFeatures: supportedFeatures, }, nil } @@ -165,7 +167,7 @@ func (c *BrokerAdminClient) GetBrokerIDs(ctx context.Context) ([]int, error) { } func (c *BrokerAdminClient) GetBootstrapAddrs() []string { - return []string{c.brokerAddr} + return []string{c.config.BrokerAddr} } func (c *BrokerAdminClient) GetTopics( @@ -283,7 +285,7 @@ func (c *BrokerAdminClient) UpdateTopicConfig( configEntries []kafka.ConfigEntry, overwrite bool, ) ([]string, error) { - if c.readOnly { + if c.config.ReadOnly { return nil, errors.New("Cannot update topic config read-only mode") } @@ -314,7 +316,7 @@ func (c *BrokerAdminClient) UpdateBrokerConfig( configEntries []kafka.ConfigEntry, overwrite bool, ) ([]string, error) { - if c.readOnly { + if c.config.ReadOnly { return nil, errors.New("Cannot update broker config read-only mode") } @@ -343,7 +345,7 @@ func (c *BrokerAdminClient) CreateTopic( ctx context.Context, config kafka.TopicConfig, ) error { - if c.readOnly { + if c.config.ReadOnly { return errors.New("Cannot create topic in read-only mode") } @@ -362,7 +364,7 @@ func (c *BrokerAdminClient) AssignPartitions( topic string, assignments []PartitionAssignment, ) error { - if c.readOnly { + if c.config.ReadOnly { return errors.New("Cannot assign partitions in read-only mode") } @@ -398,7 +400,7 @@ func (c *BrokerAdminClient) AddPartitions( topic string, newAssignments []PartitionAssignment, ) error { - if c.readOnly { + if c.config.ReadOnly { return errors.New("Cannot add partitions in read-only mode") } @@ -441,7 +443,7 @@ func (c *BrokerAdminClient) RunLeaderElection( topic string, partitions []int, ) error { - if c.readOnly { + if c.config.ReadOnly { return errors.New("Cannot run leader election in read-only mode") } diff --git a/pkg/admin/brokerclient_test.go b/pkg/admin/brokerclient_test.go index 0933ad9c..3954e155 100644 --- a/pkg/admin/brokerclient_test.go +++ b/pkg/admin/brokerclient_test.go @@ -20,7 +20,11 @@ func TestBrokerClientGetClusterID(t *testing.T) { ctx := context.Background() client, err := NewBrokerAdminClient( ctx, - BrokerAdminClientConfig{BrokerAddr: util.TestKafkaAddr()}, + BrokerAdminClientConfig{ + BrokerConnectorConfig: BrokerConnectorConfig{ + BrokerAddr: util.TestKafkaAddr(), + }, + }, ) require.NoError(t, err) @@ -37,7 +41,11 @@ func TestBrokerClientUpdateTopicConfig(t *testing.T) { ctx := context.Background() client, err := NewBrokerAdminClient( ctx, - BrokerAdminClientConfig{BrokerAddr: util.TestKafkaAddr()}, + BrokerAdminClientConfig{ + BrokerConnectorConfig: BrokerConnectorConfig{ + BrokerAddr: util.TestKafkaAddr(), + }, + }, ) require.NoError(t, err) @@ -139,7 +147,11 @@ func TestBrokerClientBrokers(t *testing.T) { ctx := context.Background() client, err := NewBrokerAdminClient( ctx, - BrokerAdminClientConfig{BrokerAddr: util.TestKafkaAddr()}, + BrokerAdminClientConfig{ + BrokerConnectorConfig: BrokerConnectorConfig{ + BrokerAddr: util.TestKafkaAddr(), + }, + }, ) require.NoError(t, err) @@ -258,7 +270,11 @@ func TestBrokerClientAddPartitions(t *testing.T) { ctx := context.Background() client, err := NewBrokerAdminClient( ctx, - BrokerAdminClientConfig{BrokerAddr: util.TestKafkaAddr()}, + BrokerAdminClientConfig{ + BrokerConnectorConfig: BrokerConnectorConfig{ + BrokerAddr: util.TestKafkaAddr(), + }, + }, ) require.NoError(t, err) @@ -322,7 +338,11 @@ func TestBrokerClientAlterAssignments(t *testing.T) { ctx := context.Background() client, err := NewBrokerAdminClient( ctx, - BrokerAdminClientConfig{BrokerAddr: util.TestKafkaAddr()}, + BrokerAdminClientConfig{ + BrokerConnectorConfig: BrokerConnectorConfig{ + BrokerAddr: util.TestKafkaAddr(), + }, + }, ) require.NoError(t, err) @@ -410,7 +430,11 @@ func TestBrokerClientRunLeaderElection(t *testing.T) { ctx := context.Background() client, err := NewBrokerAdminClient( ctx, - BrokerAdminClientConfig{BrokerAddr: util.TestKafkaAddr()}, + BrokerAdminClientConfig{ + BrokerConnectorConfig: BrokerConnectorConfig{ + BrokerAddr: util.TestKafkaAddr(), + }, + }, ) require.NoError(t, err) @@ -447,7 +471,11 @@ func TestBrokerClientGetApiVersions(t *testing.T) { ctx := context.Background() client, err := NewBrokerAdminClient( ctx, - BrokerAdminClientConfig{BrokerAddr: util.TestKafkaAddr()}, + BrokerAdminClientConfig{ + BrokerConnectorConfig: BrokerConnectorConfig{ + BrokerAddr: util.TestKafkaAddr(), + }, + }, ) require.NoError(t, err) diff --git a/pkg/admin/connector.go b/pkg/admin/connector.go new file mode 100644 index 00000000..195f4e5a --- /dev/null +++ b/pkg/admin/connector.go @@ -0,0 +1,68 @@ +package admin + +import ( + "crypto/tls" + "crypto/x509" + "fmt" + "io/ioutil" + "time" + + "github.com/segmentio/kafka-go" +) + +type BrokerConnectorConfig struct { + BrokerAddr string + UseTLS bool + CertPath string + KeyPath string + CACertPath string +} + +type BrokerConnector struct { + Config BrokerConnectorConfig + Dialer *kafka.Dialer + KafkaClient *kafka.Client +} + +func NewBrokerConnector(config BrokerConnectorConfig) (*BrokerConnector, error) { + connector := &BrokerConnector{ + Config: config, + } + + if !config.UseTLS { + connector.Dialer = kafka.DefaultDialer + } else { + cert, err := tls.LoadX509KeyPair(config.CertPath, config.KeyPath) + if err != nil { + return nil, err + } + + caCertPool := x509.NewCertPool() + caCertContents, err := ioutil.ReadFile(config.CACertPath) + if err != nil { + return nil, err + } + + if ok := caCertPool.AppendCertsFromPEM(caCertContents); !ok { + return nil, fmt.Errorf("Could not append CA certs from %s", config.CACertPath) + } + + tlsConfig := &tls.Config{ + Certificates: []tls.Certificate{cert}, + RootCAs: caCertPool, + } + connector.Dialer = &kafka.Dialer{ + Timeout: 10 * time.Second, + TLS: tlsConfig, + } + } + + connector.KafkaClient = &kafka.Client{ + Addr: kafka.TCP(config.BrokerAddr), + Transport: &kafka.Transport{ + Dial: connector.Dialer.DialFunc, + }, + } + + return connector, nil +} diff --git a/pkg/cli/cli.go b/pkg/cli/cli.go index 545b50ad..a5fe4bf3 100644 --- a/pkg/cli/cli.go +++ b/pkg/cli/cli.go @@ -28,19 +28,22 @@ const ( // CLIRunner is a utility that runs commands from either the command-line or the repl. type CLIRunner struct { - adminClient admin.Client - groupsClient *groups.Client - printer func(f string, a ...interface{}) - spinnerObj *spinner.Spinner + adminClient admin.Client + groupsClient *groups.Client + brokerClientConfig admin.BrokerClientConfig + printer func(f string, a ...interface{}) + spinnerObj *spinner.Spinner } // NewCLIRunner creates and returns a new CLIRunner instance. func NewCLIRunner( adminClient admin.Client, + brokerClientConfig admin.BrokerClientConfig, printer func(f string, a ...interface{}), showSpinner bool, ) *CLIRunner { var spinnerObj *spinner.Spinner + var err error if showSpinner { spinnerObj = spinner.New( @@ -53,12 +56,16 @@ func NewCLIRunner( } cliRunner := &CLIRunner{ - adminClient: adminClient, - printer: printer, - spinnerObj: spinnerObj, + adminClient: adminClient, + brokerClientConfig: brokerClientConfig, + printer: printer, + spinnerObj: spinnerObj, } if adminClient != nil { - cliRunner.groupsClient = groups.NewClient(adminClient.GetBootstrapAddrs()[0]) + cliRunner.groupsClient, err = groups.NewClient(brokerClientConfig) + if err != nil { + log.Warnf("Could not create groups client: %+v", err) + } } return cliRunner @@ -420,7 +427,7 @@ func (c *CLIRunner) GetOffsets(ctx context.Context, topic string) error { bounds, err := messages.GetAllPartitionBounds( ctx, - c.adminClient.GetBootstrapAddrs()[0], + c.brokerClientConfig, topic, nil, ) diff --git a/pkg/cli/repl.go b/pkg/cli/repl.go index 084eb0b6..500cb3d8 100644 --- a/pkg/cli/repl.go +++ b/pkg/cli/repl.go @@ -85,9 +85,14 @@ type Repl struct { } // NewRepl initializes and returns a Repl instance. -func NewRepl(ctx context.Context, adminClient admin.Client) (*Repl, error) { +func NewRepl( + ctx context.Context, + adminClient admin.Client, + brokerClientConfig admin.BrokerClientConfig, +) (*Repl, error) { cliRunner := NewCLIRunner( adminClient, + brokerClientConfig, func(f string, a ...interface{}) { fmt.Printf("> ") fmt.Printf(f, a...) @@ -149,7 +154,11 @@ func NewRepl(ctx context.Context, adminClient admin.Client) (*Repl, error) { } log.Debug("Loading consumer groups for auto-complete") - groupsClient := groups.NewClient(adminClient.GetBootstrapAddrs()[0]) + groupsClient, err := groups.NewClient(brokerClientConfig) + if err != nil { + return nil, err + } + groupCoordinators, err := groupsClient.GetGroups(ctx) if err != nil { log.Warnf( diff --git a/pkg/config/cluster.go b/pkg/config/cluster.go index 6604edbc..158f4060 100644 --- a/pkg/config/cluster.go +++ b/pkg/config/cluster.go @@ -137,8 +137,10 @@ func (c ClusterConfig) NewAdminClient( return admin.NewBrokerAdminClient( ctx, admin.BrokerAdminClientConfig{ - BrokerAddr: c.Spec.BootstrapAddrs[0], - ReadOnly: readOnly, + BrokerClientConfig: admin.BrokerClientConfig{ + BrokerAddr: c.Spec.BootstrapAddrs[0], + }, + ReadOnly: readOnly, }, ) } else { diff --git a/pkg/groups/client_test.go b/pkg/groups/client_test.go index 282553cb..4a1bfb34 100644 --- a/pkg/groups/client_test.go +++ b/pkg/groups/client_test.go @@ -7,6 +7,7 @@ import ( "time" "github.com/segmentio/kafka-go" + "github.com/segmentio/topicctl/pkg/admin" "github.com/segmentio/topicctl/pkg/util" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -35,7 +36,13 @@ func TestGetGroups(t *testing.T) { require.NoError(t, err) } - client := NewClient(util.TestKafkaAddr()) + client, err := NewClient( + admin.BrokerClientConfig{ + BrokerAddr: util.TestKafkaAddr(), + }, + ) + require.NoError(t, err) + groups, err := client.GetGroups(ctx) require.NoError(t, err) @@ -91,7 +98,13 @@ func TestGetLags(t *testing.T) { require.NoError(t, err) } - client := NewClient(util.TestKafkaAddr()) + client, err := NewClient( + admin.BrokerClientConfig{ + BrokerAddr: util.TestKafkaAddr(), + }, + ) + require.NoError(t, err) + lags, err := client.GetMemberLags(ctx, topicName, groupID) require.NoError(t, err) require.Equal(t, 2, len(lags)) @@ -126,8 +139,13 @@ func TestResetOffsets(t *testing.T) { require.NoError(t, err) } - client := NewClient(util.TestKafkaAddr()) - err := client.ResetOffsets( + client, err := NewClient( + admin.BrokerClientConfig{ + BrokerAddr: util.TestKafkaAddr(), + }, + ) + require.NoError(t, err) + err = client.ResetOffsets( ctx, topicName, groupID, diff --git a/pkg/groups/client.go b/pkg/groups/groups.go similarity index 82% rename from pkg/groups/client.go rename to pkg/groups/groups.go index 9122f154..76011440 100644 --- a/pkg/groups/client.go +++ b/pkg/groups/groups.go @@ -7,30 +7,16 @@ import ( "sort" "github.com/segmentio/kafka-go" + "github.com/segmentio/topicctl/pkg/admin" "github.com/segmentio/topicctl/pkg/messages" ) -// Client is a struct for getting information about consumer groups from a cluster. -type Client struct { - brokerAddr string - client *kafka.Client -} - -// NewClient creates and returns a new Client instance. -func NewClient(brokerAddr string) *Client { - return &Client{ - brokerAddr: brokerAddr, - client: &kafka.Client{ - Addr: kafka.TCP(brokerAddr), - }, - } -} - // GetGroups fetches and returns information about all consumer groups in the cluster. -func (c *Client) GetGroups( +func GetGroups( ctx context.Context, + connector *admin.BrokerConnector, ) ([]GroupCoordinator, error) { - listGroupsResp, err := c.client.ListGroups(ctx, kafka.ListGroupsRequest{}) + listGroupsResp, err := connector.KafkaClient.ListGroups(ctx, kafka.ListGroupsRequest{}) // Don't immediately fail if err is non-nil; instead, just process and return // whatever results are returned. @@ -54,11 +40,12 @@ func (c *Client) GetGroups( } // GetGroupDetails returns the details (membership, etc.) for a single consumer group. -func (c *Client) GetGroupDetails( +func GetGroupDetails( ctx context.Context, + connector *admin.BrokerConnector, groupID string, ) (*GroupDetails, error) { - describeGroupsResponse, err := c.client.DescribeGroup( + describeGroupsResponse, err := connector.KafkaClient.DescribeGroup( ctx, kafka.DescribeGroupsRequest{GroupIDs: []string{groupID}}, ) @@ -113,12 +100,13 @@ func (c *Client) GetGroupDetails( // GetMemberLags returns the lag for each partition being consumed by the argument group in the // argument topic. -func (c *Client) GetMemberLags( +func GetMemberLags( ctx context.Context, + connector *admin.BrokerConnector, topic string, groupID string, ) ([]MemberPartitionLag, error) { - groupDetails, err := c.GetGroupDetails(ctx, groupID) + groupDetails, err := GetGroupDetails(ctx, connector, groupID) if err != nil { return nil, err } @@ -129,7 +117,7 @@ func (c *Client) GetMemberLags( partitionMembers := groupDetails.PartitionMembers(topic) - offsets, err := c.client.ConsumerOffsets( + offsets, err := connector.KafkaClient.ConsumerOffsets( ctx, kafka.TopicAndGroup{ Topic: topic, GroupId: groupID, @@ -139,7 +127,7 @@ func (c *Client) GetMemberLags( return nil, err } - bounds, err := messages.GetAllPartitionBounds(ctx, c.brokerAddr, topic, offsets) + bounds, err := messages.GetAllPartitionBounds(ctx, connector, topic, offsets) if err != nil { return nil, err } @@ -167,8 +155,9 @@ func (c *Client) GetMemberLags( } // ResetOffsets updates the offsets for a given topic / group combination. -func (c *Client) ResetOffsets( +func ResetOffsets( ctx context.Context, + connector *admin.BrokerConnector, topic string, groupID string, partitionOffsets map[int]int64, @@ -176,8 +165,9 @@ func (c *Client) ResetOffsets( consumerGroup, err := kafka.NewConsumerGroup( kafka.ConsumerGroupConfig{ ID: groupID, - Brokers: []string{c.brokerAddr}, + Brokers: []string{connector.Config.BrokerAddr}, Topics: []string{topic}, + Dialer: connector.Dialer, }, ) if err != nil { diff --git a/pkg/messages/bounds.go b/pkg/messages/bounds.go index 428491a6..bb575bc3 100644 --- a/pkg/messages/bounds.go +++ b/pkg/messages/bounds.go @@ -7,6 +7,7 @@ import ( "time" "github.com/segmentio/kafka-go" + "github.com/segmentio/topicctl/pkg/admin" log "github.com/sirupsen/logrus" // Read snappy-compressed messages @@ -45,11 +46,11 @@ type Bounds struct { // is nil, the starting offset in each topic partition. func GetAllPartitionBounds( ctx context.Context, - brokerAddr string, + connector *admin.BrokerConnector, topic string, baseOffsets map[int]int64, ) ([]Bounds, error) { - conn, err := kafka.DefaultDialer.DialContext(ctx, "tcp", brokerAddr) + conn, err := connector.Dialer.DialContext(ctx, "tcp", connector.Config.BrokerAddr) if err != nil { return nil, err } @@ -90,7 +91,7 @@ func GetAllPartitionBounds( bounds, err := GetPartitionBounds( ctx, - brokerAddr, + connector, topic, nextPartition.ID, minOffset, @@ -131,7 +132,7 @@ func GetAllPartitionBounds( // this is used instead of the actual first offset. func GetPartitionBounds( ctx context.Context, - brokerAddr string, + connector *admin.BrokerConnector, topic string, partition int, minOffset int64, @@ -143,7 +144,7 @@ func GetPartitionBounds( minOffset, ) - conn, err := dialLeaderRetries(ctx, brokerAddr, topic, partition) + conn, err := dialLeaderRetries(ctx, connector, topic, partition) if err != nil { return Bounds{}, err } @@ -205,7 +206,7 @@ func GetPartitionBounds( // Use a separate connection for reading the last message. For whatever reason, // reusing the same connection with kafka-go on newer kafka versions can lead to read errors. - conn2, err := dialLeaderRetries(ctx, brokerAddr, topic, partition) + conn2, err := dialLeaderRetries(ctx, connector, topic, partition) if err != nil { return Bounds{}, err } @@ -243,7 +244,7 @@ func GetPartitionBounds( func dialLeaderRetries( ctx context.Context, - brokerAddr string, + connector *admin.BrokerConnector, topic string, partition int, ) (*kafka.Conn, error) { @@ -253,7 +254,13 @@ func dialLeaderRetries( sleepDuration := backoffInitSleepDuration for i := 0; i < maxRetries; i++ { - conn, err = kafka.DialLeader(ctx, "tcp", brokerAddr, topic, partition) + conn, err = connector.Dialer.DialLeader( + ctx, + "tcp", + connector.Config.BrokerAddr, + topic, + partition, + ) if err == nil { break } From cc7cc4c7f4b6261adb3d6b257045d4f70dafcc27 Mon Sep 17 00:00:00 2001 From: Benjamin Yolken Date: Tue, 24 Nov 2020 15:25:23 -0800 Subject: [PATCH 02/19] Keep refactoring connectors --- cmd/topicctl/subcmd/get.go | 2 +- cmd/topicctl/subcmd/repl.go | 2 +- cmd/topicctl/subcmd/reset.go | 2 +- cmd/topicctl/subcmd/tail.go | 2 +- cmd/topicctl/subcmd/tester.go | 26 +++++++++++------- pkg/admin/brokerclient.go | 8 +++--- pkg/admin/client.go | 2 +- pkg/admin/zkclient.go | 22 ++++++++------- pkg/cli/cli.go | 50 +++++++++++++++++++---------------- pkg/cli/repl.go | 9 +------ pkg/config/cluster.go | 2 +- pkg/groups/client_test.go | 25 ++++++++++-------- pkg/messages/bounds_test.go | 12 +++++++-- pkg/messages/tail.go | 30 +++++++++++---------- pkg/messages/tail_test.go | 10 ++++++- 15 files changed, 116 insertions(+), 88 deletions(-) diff --git a/cmd/topicctl/subcmd/get.go b/cmd/topicctl/subcmd/get.go index 4acaccbb..00b00f54 100644 --- a/cmd/topicctl/subcmd/get.go +++ b/cmd/topicctl/subcmd/get.go @@ -107,7 +107,7 @@ func getRun(cmd *cobra.Command, args []string) error { adminClient, clientErr = admin.NewBrokerAdminClient( ctx, admin.BrokerAdminClientConfig{ - BrokerClientConfig: admin.BrokerClientConfig{ + BrokerConnectorConfig: admin.BrokerConnectorConfig{ BrokerAddr: getConfig.brokerAddr, }, ReadOnly: true, diff --git a/cmd/topicctl/subcmd/repl.go b/cmd/topicctl/subcmd/repl.go index 17eca542..27ba4ebf 100644 --- a/cmd/topicctl/subcmd/repl.go +++ b/cmd/topicctl/subcmd/repl.go @@ -89,7 +89,7 @@ func replRun(cmd *cobra.Command, args []string) error { adminClient, clientErr = admin.NewBrokerAdminClient( ctx, admin.BrokerAdminClientConfig{ - BrokerClientConfig: admin.BrokerClientConfig{ + BrokerConnectorConfig: admin.BrokerConnectorConfig{ BrokerAddr: replConfig.brokerAddr, }, ReadOnly: true, diff --git a/cmd/topicctl/subcmd/reset.go b/cmd/topicctl/subcmd/reset.go index 27878771..b24da8f0 100644 --- a/cmd/topicctl/subcmd/reset.go +++ b/cmd/topicctl/subcmd/reset.go @@ -107,7 +107,7 @@ func resetOffsetsRun(cmd *cobra.Command, args []string) error { adminClient, clientErr = admin.NewBrokerAdminClient( ctx, admin.BrokerAdminClientConfig{ - BrokerClientConfig: admin.BrokerClientConfig{ + BrokerConnectorConfig: admin.BrokerConnectorConfig{ BrokerAddr: resetOffsetsConfig.brokerAddr, }, ReadOnly: true, diff --git a/cmd/topicctl/subcmd/tail.go b/cmd/topicctl/subcmd/tail.go index 59a47b8d..12f00545 100644 --- a/cmd/topicctl/subcmd/tail.go +++ b/cmd/topicctl/subcmd/tail.go @@ -127,7 +127,7 @@ func tailRun(cmd *cobra.Command, args []string) error { adminClient, clientErr = admin.NewBrokerAdminClient( ctx, admin.BrokerAdminClientConfig{ - BrokerClientConfig: admin.BrokerClientConfig{ + BrokerConnectorConfig: admin.BrokerConnectorConfig{ BrokerAddr: tailConfig.brokerAddr, }, ReadOnly: true, diff --git a/cmd/topicctl/subcmd/tester.go b/cmd/topicctl/subcmd/tester.go index 43c7f82c..b750f496 100644 --- a/cmd/topicctl/subcmd/tester.go +++ b/cmd/topicctl/subcmd/tester.go @@ -106,7 +106,7 @@ func testerRun(cmd *cobra.Command, args []string) error { } func runTestReader(ctx context.Context) error { - brokerAddr, err := getBrokerAddr(ctx) + brokerConnector, err := getBrokerConnector(ctx) if err != nil { return err } @@ -114,7 +114,7 @@ func runTestReader(ctx context.Context) error { log.Infof( "This will read test messages from the '%s' topic in %s using the consumer group ID '%s'", testerConfig.topic, - brokerAddr, + brokerConnector.Config.BrokerAddr, testerConfig.readConsumer, ) @@ -125,8 +125,9 @@ func runTestReader(ctx context.Context) error { reader := kafka.NewReader( kafka.ReaderConfig{ - Brokers: []string{brokerAddr}, + Brokers: []string{brokerConnector.Config.BrokerAddr}, GroupID: testerConfig.readConsumer, + Dialer: brokerConnector.Dialer, Topic: testerConfig.topic, MinBytes: 10e3, // 10KB MaxBytes: 10e6, // 10MB @@ -152,7 +153,7 @@ func runTestReader(ctx context.Context) error { } func runTestWriter(ctx context.Context) error { - brokerAddr, err := getBrokerAddr(ctx) + brokerConnector, err := getBrokerConnector(ctx) if err != nil { return err } @@ -160,7 +161,7 @@ func runTestWriter(ctx context.Context) error { log.Infof( "This will write test messages to the '%s' topic in %s at a rate of %d/sec.", testerConfig.topic, - brokerAddr, + brokerConnector.Config.BrokerAddr, testerConfig.writeRate, ) @@ -171,7 +172,8 @@ func runTestWriter(ctx context.Context) error { writer := kafka.NewWriter( kafka.WriterConfig{ - Brokers: []string{brokerAddr}, + Brokers: []string{brokerConnector.Config.BrokerAddr}, + Dialer: brokerConnector.Dialer, Topic: testerConfig.topic, Balancer: &kafka.LeastBytes{}, Async: true, @@ -210,7 +212,7 @@ func runTestWriter(ctx context.Context) error { } } -func getBrokerAddr(ctx context.Context) (string, error) { +func getBrokerConnector(ctx context.Context) (*admin.BrokerConnector, error) { if testerConfig.brokerAddr == "" { adminClient, err := admin.NewZKAdminClient( ctx, @@ -219,10 +221,14 @@ func getBrokerAddr(ctx context.Context) (string, error) { }, ) if err != nil { - return "", err + return nil, err } - return adminClient.GetBootstrapAddrs()[0], nil + return adminClient.GetBrokerConnector(), nil } else { - return testerConfig.brokerAddr, nil + return admin.NewBrokerConnector( + admin.BrokerConnectorConfig{ + BrokerAddr: testerConfig.brokerAddr, + }, + ) } } diff --git a/pkg/admin/brokerclient.go b/pkg/admin/brokerclient.go index ff11462b..22c69d56 100644 --- a/pkg/admin/brokerclient.go +++ b/pkg/admin/brokerclient.go @@ -20,7 +20,7 @@ const ( // zookeeper access. type BrokerAdminClient struct { client *kafka.Client - connector *BrokerConnector + brokerConnector *BrokerConnector config BrokerAdminClientConfig supportedFeatures SupportedFeatures } @@ -83,7 +83,7 @@ func NewBrokerAdminClient( return &BrokerAdminClient{ client: client, - connector: brokerConnector, + brokerConnector: brokerConnector, config: config, supportedFeatures: supportedFeatures, }, nil @@ -166,8 +166,8 @@ func (c *BrokerAdminClient) GetBrokerIDs(ctx context.Context) ([]int, error) { return brokerIDs, nil } -func (c *BrokerAdminClient) GetBootstrapAddrs() []string { - return []string{c.config.BrokerAddr} +func (c *BrokerAdminClient) GetBrokerConnector() *BrokerConnector { + return c.brokerConnector } func (c *BrokerAdminClient) GetTopics( diff --git a/pkg/admin/client.go b/pkg/admin/client.go index e01bffb4..334b7414 100644 --- a/pkg/admin/client.go +++ b/pkg/admin/client.go @@ -12,7 +12,7 @@ type Client interface { GetClusterID(ctx context.Context) (string, error) GetBrokers(ctx context.Context, ids []int) ([]BrokerInfo, error) GetBrokerIDs(ctx context.Context) ([]int, error) - GetBootstrapAddrs() []string + GetBrokerConnector() *BrokerConnector GetTopics( ctx context.Context, names []string, diff --git a/pkg/admin/zkclient.go b/pkg/admin/zkclient.go index 7eb26335..7177b9ff 100644 --- a/pkg/admin/zkclient.go +++ b/pkg/admin/zkclient.go @@ -44,11 +44,12 @@ var ( // zookeeper access. Most interactions are done via the latter, but a few (e.g., creating topics or // getting the controller address) are done via the broker API instead. type ZKAdminClient struct { - zkClient zk.Client - zkPrefix string - bootstrapAddrs []string - sess *session.Session - readOnly bool + zkClient zk.Client + zkPrefix string + bootstrapAddrs []string + brokerConnector *BrokerConnector + sess *session.Session + readOnly bool } var _ Client = (*ZKAdminClient)(nil) @@ -132,6 +133,11 @@ func NewZKAdminClient( } client.bootstrapAddrs = bootstrapAddrs + client.brokerConnector, err = NewBrokerConnector( + BrokerConnectorConfig{ + BrokerAddr: bootstrapAddrs[0], + }, + ) return client, nil } @@ -287,10 +293,8 @@ func (c *ZKAdminClient) GetBrokerIDs(ctx context.Context) ([]int, error) { return brokerIDs, nil } -// GetBootstrapAddrs returns the stored value of the bootstrapAddrs -// parameter so it can be used by the messages package. -func (c *ZKAdminClient) GetBootstrapAddrs() []string { - return c.bootstrapAddrs +func (c *ZKAdminClient) GetBrokerConnector() *BrokerConnector { + return c.brokerConnector } // GetTopics gets information about one or more cluster topics from zookeeper. diff --git a/pkg/cli/cli.go b/pkg/cli/cli.go index a5fe4bf3..17885ba8 100644 --- a/pkg/cli/cli.go +++ b/pkg/cli/cli.go @@ -28,22 +28,18 @@ const ( // CLIRunner is a utility that runs commands from either the command-line or the repl. type CLIRunner struct { - adminClient admin.Client - groupsClient *groups.Client - brokerClientConfig admin.BrokerClientConfig - printer func(f string, a ...interface{}) - spinnerObj *spinner.Spinner + adminClient admin.Client + printer func(f string, a ...interface{}) + spinnerObj *spinner.Spinner } // NewCLIRunner creates and returns a new CLIRunner instance. func NewCLIRunner( adminClient admin.Client, - brokerClientConfig admin.BrokerClientConfig, printer func(f string, a ...interface{}), showSpinner bool, ) *CLIRunner { var spinnerObj *spinner.Spinner - var err error if showSpinner { spinnerObj = spinner.New( @@ -56,16 +52,9 @@ func NewCLIRunner( } cliRunner := &CLIRunner{ - adminClient: adminClient, - brokerClientConfig: brokerClientConfig, - printer: printer, - spinnerObj: spinnerObj, - } - if adminClient != nil { - cliRunner.groupsClient, err = groups.NewClient(brokerClientConfig) - if err != nil { - log.Warnf("Could not create groups client: %+v", err) - } + adminClient: adminClient, + printer: printer, + spinnerObj: spinnerObj, } return cliRunner @@ -323,7 +312,7 @@ func (c *CLIRunner) GetConfig(ctx context.Context, brokerOrTopic string) error { func (c *CLIRunner) GetGroups(ctx context.Context) error { c.startSpinner() - groupCoordinators, err := c.groupsClient.GetGroups(ctx) + groupCoordinators, err := groups.GetGroups(ctx, c.adminClient.GetBrokerConnector()) c.stopSpinner() if err != nil { return err @@ -337,7 +326,11 @@ func (c *CLIRunner) GetGroups(ctx context.Context) error { func (c *CLIRunner) GetGroupMembers(ctx context.Context, groupID string, full bool) error { c.startSpinner() - groupDetails, err := c.groupsClient.GetGroupDetails(ctx, groupID) + groupDetails, err := groups.GetGroupDetails( + ctx, + c.adminClient.GetBrokerConnector(), + groupID, + ) c.stopSpinner() if err != nil { return err @@ -375,7 +368,12 @@ func (c *CLIRunner) GetMemberLags( return fmt.Errorf("Error fetching topic info: %+v", err) } - memberLags, err := c.groupsClient.GetMemberLags(ctx, topic, groupID) + memberLags, err := groups.GetMemberLags( + ctx, + c.adminClient.GetBrokerConnector(), + topic, + groupID, + ) c.stopSpinner() if err != nil { @@ -427,7 +425,7 @@ func (c *CLIRunner) GetOffsets(ctx context.Context, topic string) error { bounds, err := messages.GetAllPartitionBounds( ctx, - c.brokerClientConfig, + c.adminClient.GetBrokerConnector(), topic, nil, ) @@ -483,7 +481,13 @@ func (c *CLIRunner) ResetOffsets( partitionOffsets map[int]int64, ) error { c.startSpinner() - err := c.groupsClient.ResetOffsets(ctx, topic, groupID, partitionOffsets) + err := groups.ResetOffsets( + ctx, + c.adminClient.GetBrokerConnector(), + topic, + groupID, + partitionOffsets, + ) c.stopSpinner() if err != nil { return err @@ -516,7 +520,7 @@ func (c *CLIRunner) Tail( log.Debugf("Tailing partitions %+v", partitions) tailer := messages.NewTopicTailer( - c.adminClient.GetBootstrapAddrs()[0], + c.adminClient.GetBrokerConnector(), topic, partitions, offset, diff --git a/pkg/cli/repl.go b/pkg/cli/repl.go index 500cb3d8..c83b6234 100644 --- a/pkg/cli/repl.go +++ b/pkg/cli/repl.go @@ -88,11 +88,9 @@ type Repl struct { func NewRepl( ctx context.Context, adminClient admin.Client, - brokerClientConfig admin.BrokerClientConfig, ) (*Repl, error) { cliRunner := NewCLIRunner( adminClient, - brokerClientConfig, func(f string, a ...interface{}) { fmt.Printf("> ") fmt.Printf(f, a...) @@ -154,12 +152,7 @@ func NewRepl( } log.Debug("Loading consumer groups for auto-complete") - groupsClient, err := groups.NewClient(brokerClientConfig) - if err != nil { - return nil, err - } - - groupCoordinators, err := groupsClient.GetGroups(ctx) + groupCoordinators, err := groups.GetGroups(ctx, adminClient.GetBrokerConnector()) if err != nil { log.Warnf( "Error getting groups for auto-complete: %+v; auto-complete might not be fully functional", diff --git a/pkg/config/cluster.go b/pkg/config/cluster.go index 158f4060..df6b0b1e 100644 --- a/pkg/config/cluster.go +++ b/pkg/config/cluster.go @@ -137,7 +137,7 @@ func (c ClusterConfig) NewAdminClient( return admin.NewBrokerAdminClient( ctx, admin.BrokerAdminClientConfig{ - BrokerClientConfig: admin.BrokerClientConfig{ + BrokerConnectorConfig: admin.BrokerConnectorConfig{ BrokerAddr: c.Spec.BootstrapAddrs[0], }, ReadOnly: readOnly, diff --git a/pkg/groups/client_test.go b/pkg/groups/client_test.go index 4a1bfb34..a9345c75 100644 --- a/pkg/groups/client_test.go +++ b/pkg/groups/client_test.go @@ -36,14 +36,14 @@ func TestGetGroups(t *testing.T) { require.NoError(t, err) } - client, err := NewClient( - admin.BrokerClientConfig{ + brokerConnector, err := admin.NewBrokerConnector( + admin.BrokerConnectorConfig{ BrokerAddr: util.TestKafkaAddr(), }, ) require.NoError(t, err) - groups, err := client.GetGroups(ctx) + groups, err := GetGroups(ctx, brokerConnector) require.NoError(t, err) // There could be older groups in here, just ignore them @@ -59,7 +59,7 @@ func TestGetGroups(t *testing.T) { } require.True(t, match) - groupDetails, err := client.GetGroupDetails(ctx, groupID) + groupDetails, err := GetGroupDetails(ctx, brokerConnector, groupID) require.NoError(t, err) assert.Equal(t, groupID, groupDetails.GroupID) assert.Equal(t, "Stable", groupDetails.State) @@ -98,14 +98,14 @@ func TestGetLags(t *testing.T) { require.NoError(t, err) } - client, err := NewClient( - admin.BrokerClientConfig{ + brokerConnector, err := admin.NewBrokerConnector( + admin.BrokerConnectorConfig{ BrokerAddr: util.TestKafkaAddr(), }, ) require.NoError(t, err) - lags, err := client.GetMemberLags(ctx, topicName, groupID) + lags, err := GetMemberLags(ctx, brokerConnector, topicName, groupID) require.NoError(t, err) require.Equal(t, 2, len(lags)) @@ -139,14 +139,17 @@ func TestResetOffsets(t *testing.T) { require.NoError(t, err) } - client, err := NewClient( - admin.BrokerClientConfig{ + brokerConnector, err := admin.NewBrokerConnector( + admin.BrokerConnectorConfig{ BrokerAddr: util.TestKafkaAddr(), }, ) require.NoError(t, err) - err = client.ResetOffsets( + + require.NoError(t, err) + err = ResetOffsets( ctx, + brokerConnector, topicName, groupID, map[int]int64{ @@ -156,7 +159,7 @@ func TestResetOffsets(t *testing.T) { ) require.NoError(t, err) - lags, err := client.GetMemberLags(ctx, topicName, groupID) + lags, err := GetMemberLags(ctx, brokerConnector, topicName, groupID) require.NoError(t, err) require.Equal(t, 2, len(lags)) diff --git a/pkg/messages/bounds_test.go b/pkg/messages/bounds_test.go index 88df155c..5aca1c12 100644 --- a/pkg/messages/bounds_test.go +++ b/pkg/messages/bounds_test.go @@ -7,6 +7,7 @@ import ( "time" "github.com/segmentio/kafka-go" + "github.com/segmentio/topicctl/pkg/admin" "github.com/segmentio/topicctl/pkg/util" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -53,7 +54,14 @@ func TestGetAllPartitionBounds(t *testing.T) { err = writer.WriteMessages(ctx, messages...) require.NoError(t, err) - bounds, err := GetAllPartitionBounds(ctx, util.TestKafkaAddr(), topicName, nil) + brokerConnector, err := admin.NewBrokerConnector( + admin.BrokerConnectorConfig{ + BrokerAddr: util.TestKafkaAddr(), + }, + ) + require.NoError(t, err) + + bounds, err := GetAllPartitionBounds(ctx, brokerConnector, topicName, nil) assert.Nil(t, err) // The first partition gets 3 messages @@ -69,7 +77,7 @@ func TestGetAllPartitionBounds(t *testing.T) { boundsWithOffsets, err := GetAllPartitionBounds( ctx, - util.TestKafkaAddr(), + brokerConnector, topicName, map[int]int64{ 0: 1, diff --git a/pkg/messages/tail.go b/pkg/messages/tail.go index 71efbae9..63c9b3cf 100644 --- a/pkg/messages/tail.go +++ b/pkg/messages/tail.go @@ -10,6 +10,7 @@ import ( "github.com/fatih/color" "github.com/segmentio/kafka-go" + "github.com/segmentio/topicctl/pkg/admin" "github.com/segmentio/topicctl/pkg/util" log "github.com/sirupsen/logrus" @@ -22,17 +23,17 @@ import ( // TopicTailer fetches a stream of messages from a topic. type TopicTailer struct { - brokerAddr string - topic string - partitions []int - offset int64 - minBytes int - maxBytes int + brokerConnector *admin.BrokerConnector + topic string + partitions []int + offset int64 + minBytes int + maxBytes int } // NewTopicTailer returns a new TopicTailer instance. func NewTopicTailer( - brokerAddr string, + brokerConnector *admin.BrokerConnector, topic string, partitions []int, offset int64, @@ -40,12 +41,12 @@ func NewTopicTailer( maxBytes int, ) *TopicTailer { return &TopicTailer{ - brokerAddr: brokerAddr, - topic: topic, - partitions: partitions, - offset: offset, - minBytes: minBytes, - maxBytes: maxBytes, + brokerConnector: brokerConnector, + topic: topic, + partitions: partitions, + offset: offset, + minBytes: minBytes, + maxBytes: maxBytes, } } @@ -86,7 +87,8 @@ func (t *TopicTailer) GetMessages( for _, partition := range t.partitions { reader := kafka.NewReader( kafka.ReaderConfig{ - Brokers: []string{t.brokerAddr}, + Brokers: []string{t.brokerConnector.Config.BrokerAddr}, + Dialer: t.brokerConnector.Dialer, Topic: t.topic, Partition: partition, MinBytes: t.minBytes, diff --git a/pkg/messages/tail_test.go b/pkg/messages/tail_test.go index c46c81b8..6f2de84e 100644 --- a/pkg/messages/tail_test.go +++ b/pkg/messages/tail_test.go @@ -7,6 +7,7 @@ import ( "time" "github.com/segmentio/kafka-go" + "github.com/segmentio/topicctl/pkg/admin" "github.com/segmentio/topicctl/pkg/util" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -54,8 +55,15 @@ func TestTailerGetMessages(t *testing.T) { err = writer.WriteMessages(ctx, messages...) require.NoError(t, err) + brokerConnector, err := admin.NewBrokerConnector( + admin.BrokerConnectorConfig{ + BrokerAddr: util.TestKafkaAddr(), + }, + ) + require.NoError(t, err) + tailer := NewTopicTailer( - util.TestKafkaAddr(), + brokerConnector, topicName, []int{0, 1, 2, 3}, kafka.FirstOffset, From 3e2c52080de69ad960f8708068f95dba9a4dca40 Mon Sep 17 00:00:00 2001 From: Benjamin Yolken Date: Tue, 24 Nov 2020 15:44:59 -0800 Subject: [PATCH 03/19] Switch to connectors in more places --- cmd/topicctl/subcmd/get.go | 2 +- cmd/topicctl/subcmd/repl.go | 2 +- cmd/topicctl/subcmd/reset.go | 2 +- cmd/topicctl/subcmd/tail.go | 2 +- cmd/topicctl/subcmd/tester.go | 24 ++--- pkg/admin/brokerclient.go | 14 +-- pkg/admin/brokerclient_test.go | 14 +-- pkg/admin/client.go | 2 +- pkg/admin/connector.go | 10 +-- pkg/admin/zkclient.go | 35 +++----- pkg/cli/cli.go | 12 +-- pkg/cli/repl.go | 2 +- pkg/config/cluster.go | 2 +- pkg/groups/groups.go | 13 +-- pkg/groups/{client_test.go => groups_test.go} | 89 ++++++++++--------- pkg/messages/bounds.go | 6 +- pkg/messages/bounds_test.go | 36 ++++---- pkg/messages/tail.go | 30 +++---- pkg/messages/tail_test.go | 33 +++---- pkg/util/testing.go | 28 ------ 20 files changed, 165 insertions(+), 193 deletions(-) rename pkg/groups/{client_test.go => groups_test.go} (71%) diff --git a/cmd/topicctl/subcmd/get.go b/cmd/topicctl/subcmd/get.go index 00b00f54..5e518b4e 100644 --- a/cmd/topicctl/subcmd/get.go +++ b/cmd/topicctl/subcmd/get.go @@ -107,7 +107,7 @@ func getRun(cmd *cobra.Command, args []string) error { adminClient, clientErr = admin.NewBrokerAdminClient( ctx, admin.BrokerAdminClientConfig{ - BrokerConnectorConfig: admin.BrokerConnectorConfig{ + ConnectorConfig: admin.ConnectorConfig{ BrokerAddr: getConfig.brokerAddr, }, ReadOnly: true, diff --git a/cmd/topicctl/subcmd/repl.go b/cmd/topicctl/subcmd/repl.go index 27ba4ebf..3d67d97a 100644 --- a/cmd/topicctl/subcmd/repl.go +++ b/cmd/topicctl/subcmd/repl.go @@ -89,7 +89,7 @@ func replRun(cmd *cobra.Command, args []string) error { adminClient, clientErr = admin.NewBrokerAdminClient( ctx, admin.BrokerAdminClientConfig{ - BrokerConnectorConfig: admin.BrokerConnectorConfig{ + ConnectorConfig: admin.ConnectorConfig{ BrokerAddr: replConfig.brokerAddr, }, ReadOnly: true, diff --git a/cmd/topicctl/subcmd/reset.go b/cmd/topicctl/subcmd/reset.go index b24da8f0..fbd43946 100644 --- a/cmd/topicctl/subcmd/reset.go +++ b/cmd/topicctl/subcmd/reset.go @@ -107,7 +107,7 @@ func resetOffsetsRun(cmd *cobra.Command, args []string) error { adminClient, clientErr = admin.NewBrokerAdminClient( ctx, admin.BrokerAdminClientConfig{ - BrokerConnectorConfig: admin.BrokerConnectorConfig{ + ConnectorConfig: admin.ConnectorConfig{ BrokerAddr: resetOffsetsConfig.brokerAddr, }, ReadOnly: true, diff --git a/cmd/topicctl/subcmd/tail.go b/cmd/topicctl/subcmd/tail.go index 12f00545..db444b8d 100644 --- a/cmd/topicctl/subcmd/tail.go +++ b/cmd/topicctl/subcmd/tail.go @@ -127,7 +127,7 @@ func tailRun(cmd *cobra.Command, args []string) error { adminClient, clientErr = admin.NewBrokerAdminClient( ctx, admin.BrokerAdminClientConfig{ - BrokerConnectorConfig: admin.BrokerConnectorConfig{ + ConnectorConfig: admin.ConnectorConfig{ BrokerAddr: tailConfig.brokerAddr, }, ReadOnly: true, diff --git a/cmd/topicctl/subcmd/tester.go b/cmd/topicctl/subcmd/tester.go index b750f496..3cac468b 100644 --- a/cmd/topicctl/subcmd/tester.go +++ b/cmd/topicctl/subcmd/tester.go @@ -106,7 +106,7 @@ func testerRun(cmd *cobra.Command, args []string) error { } func runTestReader(ctx context.Context) error { - brokerConnector, err := getBrokerConnector(ctx) + connector, err := getConnector(ctx) if err != nil { return err } @@ -114,7 +114,7 @@ func runTestReader(ctx context.Context) error { log.Infof( "This will read test messages from the '%s' topic in %s using the consumer group ID '%s'", testerConfig.topic, - brokerConnector.Config.BrokerAddr, + connector.Config.BrokerAddr, testerConfig.readConsumer, ) @@ -125,9 +125,9 @@ func runTestReader(ctx context.Context) error { reader := kafka.NewReader( kafka.ReaderConfig{ - Brokers: []string{brokerConnector.Config.BrokerAddr}, + Brokers: []string{connector.Config.BrokerAddr}, GroupID: testerConfig.readConsumer, - Dialer: brokerConnector.Dialer, + Dialer: connector.Dialer, Topic: testerConfig.topic, MinBytes: 10e3, // 10KB MaxBytes: 10e6, // 10MB @@ -153,7 +153,7 @@ func runTestReader(ctx context.Context) error { } func runTestWriter(ctx context.Context) error { - brokerConnector, err := getBrokerConnector(ctx) + connector, err := getConnector(ctx) if err != nil { return err } @@ -161,7 +161,7 @@ func runTestWriter(ctx context.Context) error { log.Infof( "This will write test messages to the '%s' topic in %s at a rate of %d/sec.", testerConfig.topic, - brokerConnector.Config.BrokerAddr, + connector.Config.BrokerAddr, testerConfig.writeRate, ) @@ -172,8 +172,8 @@ func runTestWriter(ctx context.Context) error { writer := kafka.NewWriter( kafka.WriterConfig{ - Brokers: []string{brokerConnector.Config.BrokerAddr}, - Dialer: brokerConnector.Dialer, + Brokers: []string{connector.Config.BrokerAddr}, + Dialer: connector.Dialer, Topic: testerConfig.topic, Balancer: &kafka.LeastBytes{}, Async: true, @@ -212,7 +212,7 @@ func runTestWriter(ctx context.Context) error { } } -func getBrokerConnector(ctx context.Context) (*admin.BrokerConnector, error) { +func getConnector(ctx context.Context) (*admin.Connector, error) { if testerConfig.brokerAddr == "" { adminClient, err := admin.NewZKAdminClient( ctx, @@ -223,10 +223,10 @@ func getBrokerConnector(ctx context.Context) (*admin.BrokerConnector, error) { if err != nil { return nil, err } - return adminClient.GetBrokerConnector(), nil + return adminClient.GetConnector(), nil } else { - return admin.NewBrokerConnector( - admin.BrokerConnectorConfig{ + return admin.NewConnector( + admin.ConnectorConfig{ BrokerAddr: testerConfig.brokerAddr, }, ) diff --git a/pkg/admin/brokerclient.go b/pkg/admin/brokerclient.go index 22c69d56..aad8a94b 100644 --- a/pkg/admin/brokerclient.go +++ b/pkg/admin/brokerclient.go @@ -20,7 +20,7 @@ const ( // zookeeper access. type BrokerAdminClient struct { client *kafka.Client - brokerConnector *BrokerConnector + connector *Connector config BrokerAdminClientConfig supportedFeatures SupportedFeatures } @@ -28,7 +28,7 @@ type BrokerAdminClient struct { var _ Client = (*BrokerAdminClient)(nil) type BrokerAdminClientConfig struct { - BrokerConnectorConfig + ConnectorConfig ReadOnly bool } @@ -36,11 +36,11 @@ func NewBrokerAdminClient( ctx context.Context, config BrokerAdminClientConfig, ) (*BrokerAdminClient, error) { - brokerConnector, err := NewBrokerConnector(config.BrokerConnectorConfig) + connector, err := NewConnector(config.ConnectorConfig) if err != nil { return nil, err } - client := brokerConnector.KafkaClient + client := connector.KafkaClient apiVersions, err := client.ApiVersions(ctx, kafka.ApiVersionsRequest{}) if err != nil { @@ -83,7 +83,7 @@ func NewBrokerAdminClient( return &BrokerAdminClient{ client: client, - brokerConnector: brokerConnector, + connector: connector, config: config, supportedFeatures: supportedFeatures, }, nil @@ -166,8 +166,8 @@ func (c *BrokerAdminClient) GetBrokerIDs(ctx context.Context) ([]int, error) { return brokerIDs, nil } -func (c *BrokerAdminClient) GetBrokerConnector() *BrokerConnector { - return c.brokerConnector +func (c *BrokerAdminClient) GetConnector() *Connector { + return c.connector } func (c *BrokerAdminClient) GetTopics( diff --git a/pkg/admin/brokerclient_test.go b/pkg/admin/brokerclient_test.go index 3954e155..3c5de41c 100644 --- a/pkg/admin/brokerclient_test.go +++ b/pkg/admin/brokerclient_test.go @@ -21,7 +21,7 @@ func TestBrokerClientGetClusterID(t *testing.T) { client, err := NewBrokerAdminClient( ctx, BrokerAdminClientConfig{ - BrokerConnectorConfig: BrokerConnectorConfig{ + ConnectorConfig: ConnectorConfig{ BrokerAddr: util.TestKafkaAddr(), }, }, @@ -42,7 +42,7 @@ func TestBrokerClientUpdateTopicConfig(t *testing.T) { client, err := NewBrokerAdminClient( ctx, BrokerAdminClientConfig{ - BrokerConnectorConfig: BrokerConnectorConfig{ + ConnectorConfig: ConnectorConfig{ BrokerAddr: util.TestKafkaAddr(), }, }, @@ -148,7 +148,7 @@ func TestBrokerClientBrokers(t *testing.T) { client, err := NewBrokerAdminClient( ctx, BrokerAdminClientConfig{ - BrokerConnectorConfig: BrokerConnectorConfig{ + ConnectorConfig: ConnectorConfig{ BrokerAddr: util.TestKafkaAddr(), }, }, @@ -271,7 +271,7 @@ func TestBrokerClientAddPartitions(t *testing.T) { client, err := NewBrokerAdminClient( ctx, BrokerAdminClientConfig{ - BrokerConnectorConfig: BrokerConnectorConfig{ + ConnectorConfig: ConnectorConfig{ BrokerAddr: util.TestKafkaAddr(), }, }, @@ -339,7 +339,7 @@ func TestBrokerClientAlterAssignments(t *testing.T) { client, err := NewBrokerAdminClient( ctx, BrokerAdminClientConfig{ - BrokerConnectorConfig: BrokerConnectorConfig{ + ConnectorConfig: ConnectorConfig{ BrokerAddr: util.TestKafkaAddr(), }, }, @@ -431,7 +431,7 @@ func TestBrokerClientRunLeaderElection(t *testing.T) { client, err := NewBrokerAdminClient( ctx, BrokerAdminClientConfig{ - BrokerConnectorConfig: BrokerConnectorConfig{ + ConnectorConfig: ConnectorConfig{ BrokerAddr: util.TestKafkaAddr(), }, }, @@ -472,7 +472,7 @@ func TestBrokerClientGetApiVersions(t *testing.T) { client, err := NewBrokerAdminClient( ctx, BrokerAdminClientConfig{ - BrokerConnectorConfig: BrokerConnectorConfig{ + ConnectorConfig: ConnectorConfig{ BrokerAddr: util.TestKafkaAddr(), }, }, diff --git a/pkg/admin/client.go b/pkg/admin/client.go index 334b7414..e6f884db 100644 --- a/pkg/admin/client.go +++ b/pkg/admin/client.go @@ -12,7 +12,7 @@ type Client interface { GetClusterID(ctx context.Context) (string, error) GetBrokers(ctx context.Context, ids []int) ([]BrokerInfo, error) GetBrokerIDs(ctx context.Context) ([]int, error) - GetBrokerConnector() *BrokerConnector + GetConnector() *Connector GetTopics( ctx context.Context, names []string, diff --git a/pkg/admin/connector.go b/pkg/admin/connector.go index 195f4e5a..acd8b345 100644 --- a/pkg/admin/connector.go +++ b/pkg/admin/connector.go @@ -10,7 +10,7 @@ import ( "github.com/segmentio/kafka-go" ) -type BrokerConnectorConfig struct { +type ConnectorConfig struct { BrokerAddr string UseTLS bool CertPath string @@ -18,14 +18,14 @@ type BrokerConnectorConfig struct { CACertPath string } -type BrokerConnector struct { - Config BrokerConnectorConfig +type Connector struct { + Config ConnectorConfig Dialer *kafka.Dialer KafkaClient *kafka.Client } -func NewBrokerConnector(config BrokerConnectorConfig) (*BrokerConnector, error) { - connector := &BrokerConnector{ +func NewConnector(config ConnectorConfig) (*Connector, error) { + connector := &Connector{ Config: config, } diff --git a/pkg/admin/zkclient.go b/pkg/admin/zkclient.go index 7177b9ff..20fce7ad 100644 --- a/pkg/admin/zkclient.go +++ b/pkg/admin/zkclient.go @@ -44,12 +44,12 @@ var ( // zookeeper access. Most interactions are done via the latter, but a few (e.g., creating topics or // getting the controller address) are done via the broker API instead. type ZKAdminClient struct { - zkClient zk.Client - zkPrefix string - bootstrapAddrs []string - brokerConnector *BrokerConnector - sess *session.Session - readOnly bool + zkClient zk.Client + zkPrefix string + bootstrapAddrs []string + Connector *Connector + sess *session.Session + readOnly bool } var _ Client = (*ZKAdminClient)(nil) @@ -133,8 +133,8 @@ func NewZKAdminClient( } client.bootstrapAddrs = bootstrapAddrs - client.brokerConnector, err = NewBrokerConnector( - BrokerConnectorConfig{ + client.Connector, err = NewConnector( + ConnectorConfig{ BrokerAddr: bootstrapAddrs[0], }, ) @@ -293,8 +293,8 @@ func (c *ZKAdminClient) GetBrokerIDs(ctx context.Context) ([]int, error) { return brokerIDs, nil } -func (c *ZKAdminClient) GetBrokerConnector() *BrokerConnector { - return c.brokerConnector +func (c *ZKAdminClient) GetConnector() *Connector { + return c.Connector } // GetTopics gets information about one or more cluster topics from zookeeper. @@ -568,20 +568,13 @@ func (c *ZKAdminClient) CreateTopic( return errors.New("Cannot create topic in read-only mode") } - controllerAddr, err := c.getControllerAddr(ctx) - if err != nil { - return err + req := kafka.CreateTopicsRequest{ + Topics: []kafka.TopicConfig{config}, } - - conn, err := kafka.DefaultDialer.DialContext(ctx, "tcp", controllerAddr) - if err != nil { - return err - } - defer conn.Close() - log.Debugf("Creating topic with config %+v", config) - return conn.CreateTopics(config) + _, err := c.Connector.KafkaClient.CreateTopics(ctx, &req) + return err } // AssignPartitions notifies the cluster to begin a partition reassignment. diff --git a/pkg/cli/cli.go b/pkg/cli/cli.go index 17885ba8..b2070edd 100644 --- a/pkg/cli/cli.go +++ b/pkg/cli/cli.go @@ -312,7 +312,7 @@ func (c *CLIRunner) GetConfig(ctx context.Context, brokerOrTopic string) error { func (c *CLIRunner) GetGroups(ctx context.Context) error { c.startSpinner() - groupCoordinators, err := groups.GetGroups(ctx, c.adminClient.GetBrokerConnector()) + groupCoordinators, err := groups.GetGroups(ctx, c.adminClient.GetConnector()) c.stopSpinner() if err != nil { return err @@ -328,7 +328,7 @@ func (c *CLIRunner) GetGroupMembers(ctx context.Context, groupID string, full bo groupDetails, err := groups.GetGroupDetails( ctx, - c.adminClient.GetBrokerConnector(), + c.adminClient.GetConnector(), groupID, ) c.stopSpinner() @@ -370,7 +370,7 @@ func (c *CLIRunner) GetMemberLags( memberLags, err := groups.GetMemberLags( ctx, - c.adminClient.GetBrokerConnector(), + c.adminClient.GetConnector(), topic, groupID, ) @@ -425,7 +425,7 @@ func (c *CLIRunner) GetOffsets(ctx context.Context, topic string) error { bounds, err := messages.GetAllPartitionBounds( ctx, - c.adminClient.GetBrokerConnector(), + c.adminClient.GetConnector(), topic, nil, ) @@ -483,7 +483,7 @@ func (c *CLIRunner) ResetOffsets( c.startSpinner() err := groups.ResetOffsets( ctx, - c.adminClient.GetBrokerConnector(), + c.adminClient.GetConnector(), topic, groupID, partitionOffsets, @@ -520,7 +520,7 @@ func (c *CLIRunner) Tail( log.Debugf("Tailing partitions %+v", partitions) tailer := messages.NewTopicTailer( - c.adminClient.GetBrokerConnector(), + c.adminClient.GetConnector(), topic, partitions, offset, diff --git a/pkg/cli/repl.go b/pkg/cli/repl.go index c83b6234..43fd711f 100644 --- a/pkg/cli/repl.go +++ b/pkg/cli/repl.go @@ -152,7 +152,7 @@ func NewRepl( } log.Debug("Loading consumer groups for auto-complete") - groupCoordinators, err := groups.GetGroups(ctx, adminClient.GetBrokerConnector()) + groupCoordinators, err := groups.GetGroups(ctx, adminClient.GetConnector()) if err != nil { log.Warnf( "Error getting groups for auto-complete: %+v; auto-complete might not be fully functional", diff --git a/pkg/config/cluster.go b/pkg/config/cluster.go index df6b0b1e..1ae1650c 100644 --- a/pkg/config/cluster.go +++ b/pkg/config/cluster.go @@ -137,7 +137,7 @@ func (c ClusterConfig) NewAdminClient( return admin.NewBrokerAdminClient( ctx, admin.BrokerAdminClientConfig{ - BrokerConnectorConfig: admin.BrokerConnectorConfig{ + ConnectorConfig: admin.ConnectorConfig{ BrokerAddr: c.Spec.BootstrapAddrs[0], }, ReadOnly: readOnly, diff --git a/pkg/groups/groups.go b/pkg/groups/groups.go index 76011440..5cac948f 100644 --- a/pkg/groups/groups.go +++ b/pkg/groups/groups.go @@ -14,9 +14,12 @@ import ( // GetGroups fetches and returns information about all consumer groups in the cluster. func GetGroups( ctx context.Context, - connector *admin.BrokerConnector, + connector *admin.Connector, ) ([]GroupCoordinator, error) { - listGroupsResp, err := connector.KafkaClient.ListGroups(ctx, kafka.ListGroupsRequest{}) + listGroupsResp, err := connector.KafkaClient.ListGroups( + ctx, + kafka.ListGroupsRequest{}, + ) // Don't immediately fail if err is non-nil; instead, just process and return // whatever results are returned. @@ -42,7 +45,7 @@ func GetGroups( // GetGroupDetails returns the details (membership, etc.) for a single consumer group. func GetGroupDetails( ctx context.Context, - connector *admin.BrokerConnector, + connector *admin.Connector, groupID string, ) (*GroupDetails, error) { describeGroupsResponse, err := connector.KafkaClient.DescribeGroup( @@ -102,7 +105,7 @@ func GetGroupDetails( // argument topic. func GetMemberLags( ctx context.Context, - connector *admin.BrokerConnector, + connector *admin.Connector, topic string, groupID string, ) ([]MemberPartitionLag, error) { @@ -157,7 +160,7 @@ func GetMemberLags( // ResetOffsets updates the offsets for a given topic / group combination. func ResetOffsets( ctx context.Context, - connector *admin.BrokerConnector, + connector *admin.Connector, topic string, groupID string, partitionOffsets map[int]int64, diff --git a/pkg/groups/client_test.go b/pkg/groups/groups_test.go similarity index 71% rename from pkg/groups/client_test.go rename to pkg/groups/groups_test.go index a9345c75..f0e431d9 100644 --- a/pkg/groups/client_test.go +++ b/pkg/groups/groups_test.go @@ -15,12 +15,18 @@ import ( func TestGetGroups(t *testing.T) { ctx := context.Background() - topicName := createTestTopic(ctx, t) + connector, err := admin.NewConnector(admin.ConnectorConfig{ + BrokerAddr: util.TestKafkaAddr(), + }) + require.NoError(t, err) + + topicName := createTestTopic(ctx, t, connector) groupID := fmt.Sprintf("test-group-%s", topicName) reader := kafka.NewReader( kafka.ReaderConfig{ - Brokers: []string{util.TestKafkaAddr()}, + Brokers: []string{connector.Config.BrokerAddr}, + Dialer: connector.Dialer, GroupID: groupID, Topic: topicName, MinBytes: 50, @@ -36,14 +42,7 @@ func TestGetGroups(t *testing.T) { require.NoError(t, err) } - brokerConnector, err := admin.NewBrokerConnector( - admin.BrokerConnectorConfig{ - BrokerAddr: util.TestKafkaAddr(), - }, - ) - require.NoError(t, err) - - groups, err := GetGroups(ctx, brokerConnector) + groups, err := GetGroups(ctx, connector) require.NoError(t, err) // There could be older groups in here, just ignore them @@ -59,7 +58,7 @@ func TestGetGroups(t *testing.T) { } require.True(t, match) - groupDetails, err := GetGroupDetails(ctx, brokerConnector, groupID) + groupDetails, err := GetGroupDetails(ctx, connector, groupID) require.NoError(t, err) assert.Equal(t, groupID, groupDetails.GroupID) assert.Equal(t, "Stable", groupDetails.State) @@ -77,12 +76,18 @@ func TestGetGroups(t *testing.T) { func TestGetLags(t *testing.T) { ctx := context.Background() - topicName := createTestTopic(ctx, t) + connector, err := admin.NewConnector(admin.ConnectorConfig{ + BrokerAddr: util.TestKafkaAddr(), + }) + require.NoError(t, err) + + topicName := createTestTopic(ctx, t, connector) groupID := fmt.Sprintf("test-group-%s", topicName) reader := kafka.NewReader( kafka.ReaderConfig{ - Brokers: []string{util.TestKafkaAddr()}, + Brokers: []string{connector.Config.BrokerAddr}, + Dialer: connector.Dialer, GroupID: groupID, Topic: topicName, MinBytes: 50, @@ -98,14 +103,7 @@ func TestGetLags(t *testing.T) { require.NoError(t, err) } - brokerConnector, err := admin.NewBrokerConnector( - admin.BrokerConnectorConfig{ - BrokerAddr: util.TestKafkaAddr(), - }, - ) - require.NoError(t, err) - - lags, err := GetMemberLags(ctx, brokerConnector, topicName, groupID) + lags, err := GetMemberLags(ctx, connector, topicName, groupID) require.NoError(t, err) require.Equal(t, 2, len(lags)) @@ -118,12 +116,18 @@ func TestGetLags(t *testing.T) { func TestResetOffsets(t *testing.T) { ctx := context.Background() - topicName := createTestTopic(ctx, t) + connector, err := admin.NewConnector(admin.ConnectorConfig{ + BrokerAddr: util.TestKafkaAddr(), + }) + require.NoError(t, err) + + topicName := createTestTopic(ctx, t, connector) groupID := fmt.Sprintf("test-group-%s", topicName) reader := kafka.NewReader( kafka.ReaderConfig{ - Brokers: []string{util.TestKafkaAddr()}, + Brokers: []string{connector.Config.BrokerAddr}, + Dialer: connector.Dialer, GroupID: groupID, Topic: topicName, MinBytes: 50, @@ -139,17 +143,10 @@ func TestResetOffsets(t *testing.T) { require.NoError(t, err) } - brokerConnector, err := admin.NewBrokerConnector( - admin.BrokerConnectorConfig{ - BrokerAddr: util.TestKafkaAddr(), - }, - ) - require.NoError(t, err) - require.NoError(t, err) err = ResetOffsets( ctx, - brokerConnector, + connector, topicName, groupID, map[int]int64{ @@ -159,7 +156,7 @@ func TestResetOffsets(t *testing.T) { ) require.NoError(t, err) - lags, err := GetMemberLags(ctx, brokerConnector, topicName, groupID) + lags, err := GetMemberLags(ctx, connector, topicName, groupID) require.NoError(t, err) require.Equal(t, 2, len(lags)) @@ -167,17 +164,22 @@ func TestResetOffsets(t *testing.T) { assert.Equal(t, int64(1), lags[1].MemberOffset) } -func createTestTopic(ctx context.Context, t *testing.T) string { - controllerConn := util.TestKafkaContollerConn(ctx, t) - defer controllerConn.Close() - +func createTestTopic( + ctx context.Context, + t *testing.T, + connector *admin.Connector, +) string { topicName := util.RandomString("topic-groups-", 6) - - err := controllerConn.CreateTopics( - kafka.TopicConfig{ - Topic: topicName, - NumPartitions: 2, - ReplicationFactor: 1, + _, err := connector.KafkaClient.CreateTopics( + ctx, + &kafka.CreateTopicsRequest{ + Topics: []kafka.TopicConfig{ + { + Topic: topicName, + NumPartitions: 2, + ReplicationFactor: 1, + }, + }, }, ) require.NoError(t, err) @@ -185,7 +187,8 @@ func createTestTopic(ctx context.Context, t *testing.T) string { writer := kafka.NewWriter( kafka.WriterConfig{ - Brokers: []string{util.TestKafkaAddr()}, + Brokers: []string{connector.Config.BrokerAddr}, + Dialer: connector.Dialer, Topic: topicName, BatchSize: 10, }, diff --git a/pkg/messages/bounds.go b/pkg/messages/bounds.go index bb575bc3..8ad973f6 100644 --- a/pkg/messages/bounds.go +++ b/pkg/messages/bounds.go @@ -46,7 +46,7 @@ type Bounds struct { // is nil, the starting offset in each topic partition. func GetAllPartitionBounds( ctx context.Context, - connector *admin.BrokerConnector, + connector *admin.Connector, topic string, baseOffsets map[int]int64, ) ([]Bounds, error) { @@ -132,7 +132,7 @@ func GetAllPartitionBounds( // this is used instead of the actual first offset. func GetPartitionBounds( ctx context.Context, - connector *admin.BrokerConnector, + connector *admin.Connector, topic string, partition int, minOffset int64, @@ -244,7 +244,7 @@ func GetPartitionBounds( func dialLeaderRetries( ctx context.Context, - connector *admin.BrokerConnector, + connector *admin.Connector, topic string, partition int, ) (*kafka.Conn, error) { diff --git a/pkg/messages/bounds_test.go b/pkg/messages/bounds_test.go index 5aca1c12..ed768903 100644 --- a/pkg/messages/bounds_test.go +++ b/pkg/messages/bounds_test.go @@ -15,16 +15,22 @@ import ( func TestGetAllPartitionBounds(t *testing.T) { ctx := context.Background() - controllerConn := util.TestKafkaContollerConn(ctx, t) - defer controllerConn.Close() + connector, err := admin.NewConnector(admin.ConnectorConfig{ + BrokerAddr: util.TestKafkaAddr(), + }) + require.NoError(t, err) topicName := util.RandomString("topic-bounds-", 6) - - err := controllerConn.CreateTopics( - kafka.TopicConfig{ - Topic: topicName, - NumPartitions: 4, - ReplicationFactor: 1, + _, err = connector.KafkaClient.CreateTopics( + ctx, + &kafka.CreateTopicsRequest{ + Topics: []kafka.TopicConfig{ + { + Topic: topicName, + NumPartitions: 4, + ReplicationFactor: 1, + }, + }, }, ) require.NoError(t, err) @@ -32,7 +38,8 @@ func TestGetAllPartitionBounds(t *testing.T) { writer := kafka.NewWriter( kafka.WriterConfig{ - Brokers: []string{util.TestKafkaAddr()}, + Brokers: []string{connector.Config.BrokerAddr}, + Dialer: connector.Dialer, Topic: topicName, Balancer: &kafka.RoundRobin{}, }, @@ -54,14 +61,7 @@ func TestGetAllPartitionBounds(t *testing.T) { err = writer.WriteMessages(ctx, messages...) require.NoError(t, err) - brokerConnector, err := admin.NewBrokerConnector( - admin.BrokerConnectorConfig{ - BrokerAddr: util.TestKafkaAddr(), - }, - ) - require.NoError(t, err) - - bounds, err := GetAllPartitionBounds(ctx, brokerConnector, topicName, nil) + bounds, err := GetAllPartitionBounds(ctx, connector, topicName, nil) assert.Nil(t, err) // The first partition gets 3 messages @@ -77,7 +77,7 @@ func TestGetAllPartitionBounds(t *testing.T) { boundsWithOffsets, err := GetAllPartitionBounds( ctx, - brokerConnector, + connector, topicName, map[int]int64{ 0: 1, diff --git a/pkg/messages/tail.go b/pkg/messages/tail.go index 63c9b3cf..481d185f 100644 --- a/pkg/messages/tail.go +++ b/pkg/messages/tail.go @@ -23,17 +23,17 @@ import ( // TopicTailer fetches a stream of messages from a topic. type TopicTailer struct { - brokerConnector *admin.BrokerConnector - topic string - partitions []int - offset int64 - minBytes int - maxBytes int + Connector *admin.Connector + topic string + partitions []int + offset int64 + minBytes int + maxBytes int } // NewTopicTailer returns a new TopicTailer instance. func NewTopicTailer( - brokerConnector *admin.BrokerConnector, + Connector *admin.Connector, topic string, partitions []int, offset int64, @@ -41,12 +41,12 @@ func NewTopicTailer( maxBytes int, ) *TopicTailer { return &TopicTailer{ - brokerConnector: brokerConnector, - topic: topic, - partitions: partitions, - offset: offset, - minBytes: minBytes, - maxBytes: maxBytes, + Connector: Connector, + topic: topic, + partitions: partitions, + offset: offset, + minBytes: minBytes, + maxBytes: maxBytes, } } @@ -87,8 +87,8 @@ func (t *TopicTailer) GetMessages( for _, partition := range t.partitions { reader := kafka.NewReader( kafka.ReaderConfig{ - Brokers: []string{t.brokerConnector.Config.BrokerAddr}, - Dialer: t.brokerConnector.Dialer, + Brokers: []string{t.Connector.Config.BrokerAddr}, + Dialer: t.Connector.Dialer, Topic: t.topic, Partition: partition, MinBytes: t.minBytes, diff --git a/pkg/messages/tail_test.go b/pkg/messages/tail_test.go index 6f2de84e..8011d982 100644 --- a/pkg/messages/tail_test.go +++ b/pkg/messages/tail_test.go @@ -17,15 +17,22 @@ func TestTailerGetMessages(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - controllerConn := util.TestKafkaContollerConn(ctx, t) + connector, err := admin.NewConnector(admin.ConnectorConfig{ + BrokerAddr: util.TestKafkaAddr(), + }) + require.NoError(t, err) topicName := util.RandomString("topic-tail-", 6) - - err := controllerConn.CreateTopics( - kafka.TopicConfig{ - Topic: topicName, - NumPartitions: 4, - ReplicationFactor: 1, + _, err = connector.KafkaClient.CreateTopics( + ctx, + &kafka.CreateTopicsRequest{ + Topics: []kafka.TopicConfig{ + { + Topic: topicName, + NumPartitions: 4, + ReplicationFactor: 1, + }, + }, }, ) require.NoError(t, err) @@ -33,7 +40,8 @@ func TestTailerGetMessages(t *testing.T) { writer := kafka.NewWriter( kafka.WriterConfig{ - Brokers: []string{util.TestKafkaAddr()}, + Brokers: []string{connector.Config.BrokerAddr}, + Dialer: connector.Dialer, Topic: topicName, Balancer: &kafka.RoundRobin{}, }, @@ -55,15 +63,8 @@ func TestTailerGetMessages(t *testing.T) { err = writer.WriteMessages(ctx, messages...) require.NoError(t, err) - brokerConnector, err := admin.NewBrokerConnector( - admin.BrokerConnectorConfig{ - BrokerAddr: util.TestKafkaAddr(), - }, - ) - require.NoError(t, err) - tailer := NewTopicTailer( - brokerConnector, + connector, topicName, []int{0, 1, 2, 3}, kafka.FirstOffset, diff --git a/pkg/util/testing.go b/pkg/util/testing.go index 0d4db00b..33301be6 100644 --- a/pkg/util/testing.go +++ b/pkg/util/testing.go @@ -1,14 +1,12 @@ package util import ( - "context" "fmt" "math/rand" "os" "testing" "time" - "github.com/segmentio/kafka-go" "github.com/stretchr/testify/require" ) @@ -40,32 +38,6 @@ func TestKafkaAddr() string { return testKafkaAddr } -// TestKafkaConn returns a kafka-go connection for unit testing purposes. -func TestKafkaConn(ctx context.Context, t *testing.T) *kafka.Conn { - conn, err := kafka.DefaultDialer.DialContext(ctx, "tcp", TestKafkaAddr()) - require.NoError(t, err) - return conn -} - -// TestKafkaContollerConn returns a kafka-go connection to the cluster controller -// for unit testing purposes. -func TestKafkaContollerConn(ctx context.Context, t *testing.T) *kafka.Conn { - conn := TestKafkaConn(ctx, t) - defer conn.Close() - - broker, err := conn.Controller() - require.NoError(t, err) - - controllerConn, err := kafka.DefaultDialer.DialContext( - ctx, - "tcp", - fmt.Sprintf("%s:%d", broker.Host, broker.Port), - ) - - require.NoError(t, err) - return controllerConn -} - func CanTestBrokerAdmin() bool { value, ok := os.LookupEnv("KAFKA_TOPICS_TEST_BROKER_ADMIN") if ok && value != "" { From 8f305c6568b226af680bca9cd748ab135318606a Mon Sep 17 00:00:00 2001 From: Benjamin Yolken Date: Tue, 24 Nov 2020 16:18:04 -0800 Subject: [PATCH 04/19] Add more TLS support --- cmd/topicctl/subcmd/get.go | 28 ++++++++++++++++++++++++++++ cmd/topicctl/subcmd/repl.go | 28 ++++++++++++++++++++++++++++ cmd/topicctl/subcmd/reset.go | 28 ++++++++++++++++++++++++++++ cmd/topicctl/subcmd/tail.go | 32 +++++++++++++++++++++++++++++++- cmd/topicctl/subcmd/tester.go | 28 ++++++++++++++++++++++++++++ pkg/config/cluster.go | 18 ++++++++++++++++++ 6 files changed, 161 insertions(+), 1 deletion(-) diff --git a/cmd/topicctl/subcmd/get.go b/cmd/topicctl/subcmd/get.go index 5e518b4e..f86608ac 100644 --- a/cmd/topicctl/subcmd/get.go +++ b/cmd/topicctl/subcmd/get.go @@ -34,6 +34,9 @@ var getCmd = &cobra.Command{ type getCmdConfig struct { brokerAddr string + brokerCACert string + brokerCert string + brokerKey string clusterConfig string full bool zkAddr string @@ -49,6 +52,24 @@ func init() { "", "Broker address", ) + getCmd.Flags().StringVar( + &getConfig.brokerCACert, + "broker-ca-cert", + "", + "Path to broker client CA cert PEM file if using TLS", + ) + getCmd.Flags().StringVar( + &getConfig.brokerCert, + "broker-cert", + "", + "Path to broker client cert PEM file if using TLS", + ) + getCmd.Flags().StringVar( + &getConfig.brokerKey, + "broker-key", + "", + "Path to broker client private key PEM file if using TLS", + ) getCmd.Flags().StringVar( &getConfig.clusterConfig, "cluster-config", @@ -104,11 +125,18 @@ func getRun(cmd *cobra.Command, args []string) error { } adminClient, clientErr = clusterConfig.NewAdminClient(ctx, sess, true) } else if getConfig.brokerAddr != "" { + useTLS := (getConfig.brokerCACert != "" || + getConfig.brokerCert != "" || + getConfig.brokerKey != "") adminClient, clientErr = admin.NewBrokerAdminClient( ctx, admin.BrokerAdminClientConfig{ ConnectorConfig: admin.ConnectorConfig{ BrokerAddr: getConfig.brokerAddr, + UseTLS: useTLS, + CACertPath: getConfig.brokerCACert, + CertPath: getConfig.brokerCert, + KeyPath: getConfig.brokerKey, }, ReadOnly: true, }, diff --git a/cmd/topicctl/subcmd/repl.go b/cmd/topicctl/subcmd/repl.go index 3d67d97a..112fa381 100644 --- a/cmd/topicctl/subcmd/repl.go +++ b/cmd/topicctl/subcmd/repl.go @@ -22,6 +22,9 @@ var replCmd = &cobra.Command{ type replCmdConfig struct { brokerAddr string + brokerCACert string + brokerCert string + brokerKey string clusterConfig string zkAddr string zkPrefix string @@ -36,6 +39,24 @@ func init() { "", "Broker address", ) + replCmd.Flags().StringVar( + &replConfig.brokerCACert, + "broker-ca-cert", + "", + "Path to broker client CA cert PEM file if using TLS", + ) + replCmd.Flags().StringVar( + &replConfig.brokerCert, + "broker-cert", + "", + "Path to broker client cert PEM file if using TLS", + ) + replCmd.Flags().StringVar( + &replConfig.brokerKey, + "broker-key", + "", + "Path to broker client private key PEM file if using TLS", + ) replCmd.Flags().StringVar( &replConfig.clusterConfig, "cluster-config", @@ -86,11 +107,18 @@ func replRun(cmd *cobra.Command, args []string) error { } adminClient, clientErr = clusterConfig.NewAdminClient(ctx, sess, true) } else if replConfig.brokerAddr != "" { + useTLS := (replConfig.brokerCACert != "" || + replConfig.brokerCert != "" || + replConfig.brokerKey != "") adminClient, clientErr = admin.NewBrokerAdminClient( ctx, admin.BrokerAdminClientConfig{ ConnectorConfig: admin.ConnectorConfig{ BrokerAddr: replConfig.brokerAddr, + UseTLS: useTLS, + CACertPath: replConfig.brokerCACert, + CertPath: replConfig.brokerCert, + KeyPath: replConfig.brokerKey, }, ReadOnly: true, }, diff --git a/cmd/topicctl/subcmd/reset.go b/cmd/topicctl/subcmd/reset.go index fbd43946..5100908b 100644 --- a/cmd/topicctl/subcmd/reset.go +++ b/cmd/topicctl/subcmd/reset.go @@ -25,6 +25,9 @@ var resetOffsetsCmd = &cobra.Command{ type resetOffsetsCmdConfig struct { brokerAddr string + brokerCACert string + brokerCert string + brokerKey string clusterConfig string offset int64 partitions []int @@ -41,6 +44,24 @@ func init() { "", "Broker address", ) + resetOffsetsCmd.Flags().StringVar( + &resetOffsetsConfig.brokerCACert, + "broker-ca-cert", + "", + "Path to broker client CA cert PEM file if using TLS", + ) + resetOffsetsCmd.Flags().StringVar( + &resetOffsetsConfig.brokerCert, + "broker-cert", + "", + "Path to broker client cert PEM file if using TLS", + ) + resetOffsetsCmd.Flags().StringVar( + &resetOffsetsConfig.brokerKey, + "broker-key", + "", + "Path to broker client private key PEM file if using TLS", + ) resetOffsetsCmd.Flags().StringVar( &resetOffsetsConfig.clusterConfig, "cluster-config", @@ -104,11 +125,18 @@ func resetOffsetsRun(cmd *cobra.Command, args []string) error { } adminClient, clientErr = clusterConfig.NewAdminClient(ctx, nil, false) } else if resetOffsetsConfig.brokerAddr != "" { + useTLS := (resetOffsetsConfig.brokerCACert != "" || + resetOffsetsConfig.brokerCert != "" || + resetOffsetsConfig.brokerKey != "") adminClient, clientErr = admin.NewBrokerAdminClient( ctx, admin.BrokerAdminClientConfig{ ConnectorConfig: admin.ConnectorConfig{ BrokerAddr: resetOffsetsConfig.brokerAddr, + UseTLS: useTLS, + CACertPath: resetOffsetsConfig.brokerCACert, + CertPath: resetOffsetsConfig.brokerCert, + KeyPath: resetOffsetsConfig.brokerKey, }, ReadOnly: true, }, diff --git a/cmd/topicctl/subcmd/tail.go b/cmd/topicctl/subcmd/tail.go index db444b8d..2fa9600e 100644 --- a/cmd/topicctl/subcmd/tail.go +++ b/cmd/topicctl/subcmd/tail.go @@ -26,6 +26,9 @@ var tailCmd = &cobra.Command{ type tailCmdConfig struct { brokerAddr string + brokerCACert string + brokerCert string + brokerKey string clusterConfig string offset int64 partitions []int @@ -37,12 +40,31 @@ type tailCmdConfig struct { var tailConfig tailCmdConfig func init() { - tailCmd.Flags().StringVar( + tailCmd.Flags().StringVarP( &tailConfig.brokerAddr, "broker-addr", + "b", "", "Broker address", ) + tailCmd.Flags().StringVar( + &tailConfig.brokerCACert, + "broker-ca-cert", + "", + "Path to broker client CA cert PEM file if using TLS", + ) + tailCmd.Flags().StringVar( + &tailConfig.brokerCert, + "broker-cert", + "", + "Path to broker client cert PEM file if using TLS", + ) + tailCmd.Flags().StringVar( + &tailConfig.brokerKey, + "broker-key", + "", + "Path to broker client private key PEM file if using TLS", + ) tailCmd.Flags().StringVar( &tailConfig.clusterConfig, "cluster-config", @@ -124,11 +146,19 @@ func tailRun(cmd *cobra.Command, args []string) error { } adminClient, clientErr = clusterConfig.NewAdminClient(ctx, nil, true) } else if tailConfig.brokerAddr != "" { + useTLS := (resetOffsetsConfig.brokerCACert != "" || + resetOffsetsConfig.brokerCert != "" || + resetOffsetsConfig.brokerKey != "") + adminClient, clientErr = admin.NewBrokerAdminClient( ctx, admin.BrokerAdminClientConfig{ ConnectorConfig: admin.ConnectorConfig{ BrokerAddr: tailConfig.brokerAddr, + UseTLS: useTLS, + CACertPath: tailConfig.brokerCACert, + CertPath: tailConfig.brokerCert, + KeyPath: tailConfig.brokerKey, }, ReadOnly: true, }, diff --git a/cmd/topicctl/subcmd/tester.go b/cmd/topicctl/subcmd/tester.go index 3cac468b..faa45bfb 100644 --- a/cmd/topicctl/subcmd/tester.go +++ b/cmd/topicctl/subcmd/tester.go @@ -25,6 +25,9 @@ var testerCmd = &cobra.Command{ type testerCmdConfig struct { brokerAddr string + brokerCACert string + brokerCert string + brokerKey string mode string readConsumer string topic string @@ -41,6 +44,24 @@ func init() { "", "Broker address", ) + testerCmd.Flags().StringVar( + &testerConfig.brokerCACert, + "broker-ca-cert", + "", + "Path to broker client CA cert PEM file if using TLS", + ) + testerCmd.Flags().StringVar( + &testerConfig.brokerCert, + "broker-cert", + "", + "Path to broker client cert PEM file if using TLS", + ) + testerCmd.Flags().StringVar( + &testerConfig.brokerKey, + "broker-key", + "", + "Path to broker client private key PEM file if using TLS", + ) testerCmd.Flags().StringVar( &testerConfig.mode, "mode", @@ -225,9 +246,16 @@ func getConnector(ctx context.Context) (*admin.Connector, error) { } return adminClient.GetConnector(), nil } else { + useTLS := (testerConfig.brokerCACert != "" || + testerConfig.brokerCert != "" || + resetOffsetsConfig.brokerKey != "") return admin.NewConnector( admin.ConnectorConfig{ BrokerAddr: testerConfig.brokerAddr, + UseTLS: useTLS, + CACertPath: testerConfig.brokerCACert, + CertPath: testerConfig.brokerCert, + KeyPath: testerConfig.brokerKey, }, ) } diff --git a/pkg/config/cluster.go b/pkg/config/cluster.go index 1ae1650c..bf002cc8 100644 --- a/pkg/config/cluster.go +++ b/pkg/config/cluster.go @@ -78,6 +78,17 @@ type ClusterSpec struct { // UseBrokerAdmin indicates whether we should use a broker-api-based admin (if true) or // the old, zk-based admin (if false). UseBrokerAdmin bool `json:"useBrokerAdmin"` + + // BrokerAuth stores how we should authenticate broker connections, if appropriate. Only + // applies if using the broker admin. + BrokerAuth BrokerAuth `json:"brokerAuth"` +} + +type BrokerAuth struct { + UseTLS bool `json:"useTLS"` + CACertPath string `json:"caCertPath"` + CertPath string `json:"certPath"` + KeyPath string `json:"keyPath"` } // Validate evaluates whether the cluster config is valid. @@ -116,6 +127,13 @@ func (c ClusterConfig) Validate() error { ) } + if c.Spec.BrokerAuth.UseTLS && !c.Spec.UseBrokerAdmin { + err = multierror.Append( + err, + errors.New("TLS not supported unless also using broker admin"), + ) + } + return err } From c4650cbc91c13cbd4f4360e2c62bccc95ca9a7c9 Mon Sep 17 00:00:00 2001 From: Benjamin Yolken Date: Tue, 24 Nov 2020 18:31:43 -0800 Subject: [PATCH 05/19] Get TLS working --- cmd/topicctl/subcmd/get.go | 88 +++++++++++++++++++-------------- cmd/topicctl/subcmd/repl.go | 84 ++++++++++++++++++------------- cmd/topicctl/subcmd/reset.go | 93 ++++++++++++++++++++--------------- cmd/topicctl/subcmd/root.go | 33 +++++++++++++ cmd/topicctl/subcmd/tail.go | 92 +++++++++++++++++++--------------- cmd/topicctl/subcmd/tester.go | 88 ++++++++++++++++++++------------- pkg/admin/connector.go | 16 ++++-- pkg/config/cluster.go | 8 +++ 8 files changed, 314 insertions(+), 188 deletions(-) diff --git a/cmd/topicctl/subcmd/get.go b/cmd/topicctl/subcmd/get.go index f86608ac..b0efa02c 100644 --- a/cmd/topicctl/subcmd/get.go +++ b/cmd/topicctl/subcmd/get.go @@ -2,7 +2,6 @@ package subcmd import ( "context" - "errors" "fmt" "os" "strings" @@ -34,11 +33,13 @@ var getCmd = &cobra.Command{ type getCmdConfig struct { brokerAddr string - brokerCACert string - brokerCert string - brokerKey string clusterConfig string full bool + tlsCACert string + tlsCert string + tlsKey string + tlsSkipVerify bool + tlsServerName string zkAddr string zkPrefix string } @@ -46,41 +47,54 @@ type getCmdConfig struct { var getConfig getCmdConfig func init() { - getCmd.Flags().StringVar( + getCmd.Flags().StringVarP( &getConfig.brokerAddr, "broker-addr", + "b", "", "Broker address", ) getCmd.Flags().StringVar( - &getConfig.brokerCACert, - "broker-ca-cert", + &getConfig.clusterConfig, + "cluster-config", + os.Getenv("TOPICCTL_CLUSTER_CONFIG"), + "Cluster config", + ) + getCmd.Flags().BoolVar( + &getConfig.full, + "full", + false, + "Show more full information for resources", + ) + getCmd.Flags().StringVar( + &getConfig.tlsCACert, + "tls-ca-cert", "", - "Path to broker client CA cert PEM file if using TLS", + "Path to client CA cert PEM file if using TLS", ) getCmd.Flags().StringVar( - &getConfig.brokerCert, - "broker-cert", + &getConfig.tlsCert, + "tls-cert", "", - "Path to broker client cert PEM file if using TLS", + "Path to client cert PEM file if using TLS", ) getCmd.Flags().StringVar( - &getConfig.brokerKey, - "broker-key", + &getConfig.tlsKey, + "tls-key", "", - "Path to broker client private key PEM file if using TLS", + "Path to client private key PEM file if using TLS", ) getCmd.Flags().StringVar( - &getConfig.clusterConfig, - "cluster-config", - os.Getenv("TOPICCTL_CLUSTER_CONFIG"), - "Cluster config", + &getConfig.tlsServerName, + "tls-server-name", + "", + "Server name to use for TLS cert verification", ) getCmd.Flags().BoolVar( - &getConfig.full, - "full", + &getConfig.tlsSkipVerify, + "tls-skip-verify", false, - "Show more full information for resources", + "Skip hostname verification when using TLS", ) getCmd.Flags().StringVarP( &getConfig.zkAddr, @@ -100,15 +114,15 @@ func init() { } func getPreRun(cmd *cobra.Command, args []string) error { - if getConfig.clusterConfig == "" && getConfig.zkAddr == "" && getConfig.brokerAddr == "" { - return errors.New("Must set either broker-addr, cluster-config, zk address") - } - if getConfig.clusterConfig != "" && - (getConfig.zkAddr != "" || getConfig.zkPrefix != "" || getConfig.brokerAddr != "") { - log.Warn("broker and zk flags are ignored when using cluster-config") - } - - return nil + return validateCommonFlags( + getConfig.clusterConfig, + getConfig.zkAddr, + getConfig.zkPrefix, + getConfig.brokerAddr, + getConfig.tlsCACert, + getConfig.tlsCert, + getConfig.tlsKey, + ) } func getRun(cmd *cobra.Command, args []string) error { @@ -125,18 +139,20 @@ func getRun(cmd *cobra.Command, args []string) error { } adminClient, clientErr = clusterConfig.NewAdminClient(ctx, sess, true) } else if getConfig.brokerAddr != "" { - useTLS := (getConfig.brokerCACert != "" || - getConfig.brokerCert != "" || - getConfig.brokerKey != "") + useTLS := (getConfig.tlsCACert != "" || + getConfig.tlsCert != "" || + getConfig.tlsKey != "") adminClient, clientErr = admin.NewBrokerAdminClient( ctx, admin.BrokerAdminClientConfig{ ConnectorConfig: admin.ConnectorConfig{ BrokerAddr: getConfig.brokerAddr, UseTLS: useTLS, - CACertPath: getConfig.brokerCACert, - CertPath: getConfig.brokerCert, - KeyPath: getConfig.brokerKey, + CACertPath: getConfig.tlsCACert, + CertPath: getConfig.tlsCert, + KeyPath: getConfig.tlsKey, + ServerName: getConfig.tlsServerName, + SkipVerify: getConfig.tlsSkipVerify, }, ReadOnly: true, }, diff --git a/cmd/topicctl/subcmd/repl.go b/cmd/topicctl/subcmd/repl.go index 112fa381..19bbf2ad 100644 --- a/cmd/topicctl/subcmd/repl.go +++ b/cmd/topicctl/subcmd/repl.go @@ -2,14 +2,12 @@ package subcmd import ( "context" - "errors" "os" "github.com/aws/aws-sdk-go/aws/session" "github.com/segmentio/topicctl/pkg/admin" "github.com/segmentio/topicctl/pkg/cli" "github.com/segmentio/topicctl/pkg/config" - log "github.com/sirupsen/logrus" "github.com/spf13/cobra" ) @@ -22,10 +20,12 @@ var replCmd = &cobra.Command{ type replCmdConfig struct { brokerAddr string - brokerCACert string - brokerCert string - brokerKey string clusterConfig string + tlsCACert string + tlsCert string + tlsKey string + tlsSkipVerify bool + tlsServerName string zkAddr string zkPrefix string } @@ -33,35 +33,48 @@ type replCmdConfig struct { var replConfig replCmdConfig func init() { - replCmd.Flags().StringVar( + replCmd.Flags().StringVarP( &replConfig.brokerAddr, "broker-addr", + "b", "", "Broker address", ) replCmd.Flags().StringVar( - &replConfig.brokerCACert, - "broker-ca-cert", + &replConfig.clusterConfig, + "cluster-config", + os.Getenv("TOPICCTL_CLUSTER_CONFIG"), + "Cluster config", + ) + replCmd.Flags().StringVar( + &replConfig.tlsCACert, + "tls-ca-cert", "", - "Path to broker client CA cert PEM file if using TLS", + "Path to client CA cert PEM file if using TLS", ) replCmd.Flags().StringVar( - &replConfig.brokerCert, - "broker-cert", + &replConfig.tlsCert, + "tls-cert", "", - "Path to broker client cert PEM file if using TLS", + "Path to client cert PEM file if using TLS", ) replCmd.Flags().StringVar( - &replConfig.brokerKey, - "broker-key", + &replConfig.tlsKey, + "tls-key", "", - "Path to broker client private key PEM file if using TLS", + "Path to client private key PEM file if using TLS", ) replCmd.Flags().StringVar( - &replConfig.clusterConfig, - "cluster-config", - os.Getenv("TOPICCTL_CLUSTER_CONFIG"), - "Cluster config", + &replConfig.tlsServerName, + "tls-server-name", + "", + "Server name to use for TLS cert verification", + ) + replCmd.Flags().BoolVar( + &replConfig.tlsSkipVerify, + "tls-skip-verify", + false, + "Skip hostname verification when using TLS", ) replCmd.Flags().StringVarP( &replConfig.zkAddr, @@ -81,16 +94,15 @@ func init() { } func replPreRun(cmd *cobra.Command, args []string) error { - if replConfig.clusterConfig == "" && replConfig.zkAddr == "" && - replConfig.brokerAddr == "" { - return errors.New("Must set either broker-addr, cluster-config, or zk-addr") - } - if replConfig.clusterConfig != "" && - (replConfig.zkAddr != "" || replConfig.zkPrefix != "" || replConfig.brokerAddr != "") { - log.Warn("broker and zk flags are ignored when using cluster-config") - } - - return nil + return validateCommonFlags( + replConfig.clusterConfig, + replConfig.zkAddr, + replConfig.zkPrefix, + replConfig.brokerAddr, + replConfig.tlsCACert, + replConfig.tlsCert, + replConfig.tlsKey, + ) } func replRun(cmd *cobra.Command, args []string) error { @@ -107,18 +119,20 @@ func replRun(cmd *cobra.Command, args []string) error { } adminClient, clientErr = clusterConfig.NewAdminClient(ctx, sess, true) } else if replConfig.brokerAddr != "" { - useTLS := (replConfig.brokerCACert != "" || - replConfig.brokerCert != "" || - replConfig.brokerKey != "") + useTLS := (replConfig.tlsCACert != "" || + replConfig.tlsCert != "" || + replConfig.tlsKey != "") adminClient, clientErr = admin.NewBrokerAdminClient( ctx, admin.BrokerAdminClientConfig{ ConnectorConfig: admin.ConnectorConfig{ BrokerAddr: replConfig.brokerAddr, UseTLS: useTLS, - CACertPath: replConfig.brokerCACert, - CertPath: replConfig.brokerCert, - KeyPath: replConfig.brokerKey, + CACertPath: replConfig.tlsCACert, + CertPath: replConfig.tlsCert, + KeyPath: replConfig.tlsKey, + ServerName: replConfig.tlsServerName, + SkipVerify: replConfig.tlsSkipVerify, }, ReadOnly: true, }, diff --git a/cmd/topicctl/subcmd/reset.go b/cmd/topicctl/subcmd/reset.go index 5100908b..cc898e0b 100644 --- a/cmd/topicctl/subcmd/reset.go +++ b/cmd/topicctl/subcmd/reset.go @@ -25,12 +25,14 @@ var resetOffsetsCmd = &cobra.Command{ type resetOffsetsCmdConfig struct { brokerAddr string - brokerCACert string - brokerCert string - brokerKey string clusterConfig string offset int64 partitions []int + tlsCACert string + tlsCert string + tlsKey string + tlsSkipVerify bool + tlsServerName string zkAddr string zkPrefix string } @@ -38,30 +40,13 @@ type resetOffsetsCmdConfig struct { var resetOffsetsConfig resetOffsetsCmdConfig func init() { - resetOffsetsCmd.Flags().StringVar( + resetOffsetsCmd.Flags().StringVarP( &resetOffsetsConfig.brokerAddr, "broker-addr", + "b", "", "Broker address", ) - resetOffsetsCmd.Flags().StringVar( - &resetOffsetsConfig.brokerCACert, - "broker-ca-cert", - "", - "Path to broker client CA cert PEM file if using TLS", - ) - resetOffsetsCmd.Flags().StringVar( - &resetOffsetsConfig.brokerCert, - "broker-cert", - "", - "Path to broker client cert PEM file if using TLS", - ) - resetOffsetsCmd.Flags().StringVar( - &resetOffsetsConfig.brokerKey, - "broker-key", - "", - "Path to broker client private key PEM file if using TLS", - ) resetOffsetsCmd.Flags().StringVar( &resetOffsetsConfig.clusterConfig, "cluster-config", @@ -80,6 +65,36 @@ func init() { []int{}, "Partition (defaults to all)", ) + resetOffsetsCmd.Flags().StringVar( + &resetOffsetsConfig.tlsCACert, + "tls-ca-cert", + "", + "Path to client CA cert PEM file if using TLS", + ) + resetOffsetsCmd.Flags().StringVar( + &resetOffsetsConfig.tlsCert, + "tls-cert", + "", + "Path to client cert PEM file if using TLS", + ) + resetOffsetsCmd.Flags().StringVar( + &resetOffsetsConfig.tlsKey, + "tls-key", + "", + "Path to client private key PEM file if using TLS", + ) + resetOffsetsCmd.Flags().StringVar( + &resetOffsetsConfig.tlsServerName, + "tls-server-name", + "", + "Server name to use for TLS cert verification", + ) + resetOffsetsCmd.Flags().BoolVar( + &resetOffsetsConfig.tlsSkipVerify, + "tls-skip-verify", + false, + "Skip hostname verification when using TLS", + ) resetOffsetsCmd.Flags().StringVarP( &resetOffsetsConfig.zkAddr, "zk-addr", @@ -98,17 +113,15 @@ func init() { } func resetOffsetsPreRun(cmd *cobra.Command, args []string) error { - if resetOffsetsConfig.clusterConfig == "" && resetOffsetsConfig.zkAddr == "" && - resetOffsetsConfig.brokerAddr == "" { - return errors.New("Must set either broker-addr, cluster-config, or zk-addr") - } - if resetOffsetsConfig.clusterConfig != "" && - (resetOffsetsConfig.zkAddr != "" || resetOffsetsConfig.zkPrefix != "" || - resetOffsetsConfig.brokerAddr != "") { - log.Warn("broker and zk flags are ignored when using cluster-config") - } - - return nil + return validateCommonFlags( + resetOffsetsConfig.clusterConfig, + resetOffsetsConfig.zkAddr, + resetOffsetsConfig.zkPrefix, + resetOffsetsConfig.brokerAddr, + resetOffsetsConfig.tlsCACert, + resetOffsetsConfig.tlsCert, + resetOffsetsConfig.tlsKey, + ) } func resetOffsetsRun(cmd *cobra.Command, args []string) error { @@ -125,18 +138,20 @@ func resetOffsetsRun(cmd *cobra.Command, args []string) error { } adminClient, clientErr = clusterConfig.NewAdminClient(ctx, nil, false) } else if resetOffsetsConfig.brokerAddr != "" { - useTLS := (resetOffsetsConfig.brokerCACert != "" || - resetOffsetsConfig.brokerCert != "" || - resetOffsetsConfig.brokerKey != "") + useTLS := (resetOffsetsConfig.tlsCACert != "" || + resetOffsetsConfig.tlsCert != "" || + resetOffsetsConfig.tlsKey != "") adminClient, clientErr = admin.NewBrokerAdminClient( ctx, admin.BrokerAdminClientConfig{ ConnectorConfig: admin.ConnectorConfig{ BrokerAddr: resetOffsetsConfig.brokerAddr, UseTLS: useTLS, - CACertPath: resetOffsetsConfig.brokerCACert, - CertPath: resetOffsetsConfig.brokerCert, - KeyPath: resetOffsetsConfig.brokerKey, + CACertPath: resetOffsetsConfig.tlsCACert, + CertPath: resetOffsetsConfig.tlsCert, + KeyPath: resetOffsetsConfig.tlsKey, + ServerName: resetOffsetsConfig.tlsServerName, + SkipVerify: resetOffsetsConfig.tlsSkipVerify, }, ReadOnly: true, }, diff --git a/cmd/topicctl/subcmd/root.go b/cmd/topicctl/subcmd/root.go index ad691b54..6292632f 100644 --- a/cmd/topicctl/subcmd/root.go +++ b/cmd/topicctl/subcmd/root.go @@ -1,6 +1,7 @@ package subcmd import ( + "errors" "fmt" "os" @@ -58,3 +59,35 @@ func preRun(cmd *cobra.Command, args []string) error { } return nil } + +func validateCommonFlags( + clusterConfig string, + zkAddr string, + zkPrefix string, + brokerAddr string, + tlsCACert string, + tlsCert string, + tlsKey string, +) error { + if clusterConfig == "" && zkAddr == "" && brokerAddr == "" { + return errors.New("Must set either broker-addr, cluster-config, or zk-addr") + } + if zkAddr != "" && brokerAddr != "" { + return errors.New("Cannot set both zk-addr and broker-addr") + } + if clusterConfig != "" && + (zkAddr != "" || zkPrefix != "" || brokerAddr != "" || tlsCACert != "" || + tlsCert != "" || tlsKey != "") { + log.Warn("Broker and zk flags are ignored when using cluster-config") + } + + useTLS := tlsCACert != "" || tlsCert != "" || tlsKey != "" + if useTLS && (tlsCACert == "" || tlsCert == "" || tlsKey == "") { + return errors.New("Must set tls-ca-cert, tls-cert, and tls-key if using TLS") + } + if useTLS && zkAddr != "" { + log.Warn("Auth flags are ignored accessing cluster via zookeeper") + } + + return nil +} diff --git a/cmd/topicctl/subcmd/tail.go b/cmd/topicctl/subcmd/tail.go index 2fa9600e..a3219909 100644 --- a/cmd/topicctl/subcmd/tail.go +++ b/cmd/topicctl/subcmd/tail.go @@ -2,7 +2,6 @@ package subcmd import ( "context" - "errors" "os" "os/signal" "strconv" @@ -26,13 +25,15 @@ var tailCmd = &cobra.Command{ type tailCmdConfig struct { brokerAddr string - brokerCACert string - brokerCert string - brokerKey string clusterConfig string offset int64 partitions []int raw bool + tlsCACert string + tlsCert string + tlsKey string + tlsSkipVerify bool + tlsServerName string zkAddr string zkPrefix string } @@ -47,24 +48,6 @@ func init() { "", "Broker address", ) - tailCmd.Flags().StringVar( - &tailConfig.brokerCACert, - "broker-ca-cert", - "", - "Path to broker client CA cert PEM file if using TLS", - ) - tailCmd.Flags().StringVar( - &tailConfig.brokerCert, - "broker-cert", - "", - "Path to broker client cert PEM file if using TLS", - ) - tailCmd.Flags().StringVar( - &tailConfig.brokerKey, - "broker-key", - "", - "Path to broker client private key PEM file if using TLS", - ) tailCmd.Flags().StringVar( &tailConfig.clusterConfig, "cluster-config", @@ -89,6 +72,36 @@ func init() { false, "Output raw values only", ) + tailCmd.Flags().StringVar( + &tailConfig.tlsCACert, + "tls-ca-cert", + "", + "Path to client CA cert PEM file if using TLS", + ) + tailCmd.Flags().StringVar( + &tailConfig.tlsCert, + "tls-cert", + "", + "Path to client cert PEM file if using TLS", + ) + tailCmd.Flags().StringVar( + &tailConfig.tlsKey, + "tls-key", + "", + "Path to client private key PEM file if using TLS", + ) + tailCmd.Flags().StringVar( + &tailConfig.tlsServerName, + "tls-server-name", + "", + "Server name to use for TLS cert verification", + ) + tailCmd.Flags().BoolVar( + &tailConfig.tlsSkipVerify, + "tls-skip-verify", + false, + "Skip hostname verification when using TLS", + ) tailCmd.Flags().StringVarP( &tailConfig.zkAddr, "zk-addr", @@ -112,17 +125,15 @@ func tailPreRun(cmd *cobra.Command, args []string) error { log.SetLevel(log.ErrorLevel) } - if tailConfig.clusterConfig == "" && tailConfig.zkAddr == "" && - tailConfig.brokerAddr == "" { - return errors.New("Must set either broker-addr, cluster-config, or zk-addr") - } - if tailConfig.clusterConfig != "" && - (tailConfig.zkAddr != "" || tailConfig.zkPrefix != "" || - tailConfig.brokerAddr != "") { - log.Warn("broker and zk flags are ignored when using cluster-config") - } - - return nil + return validateCommonFlags( + tailConfig.clusterConfig, + tailConfig.zkAddr, + tailConfig.zkPrefix, + tailConfig.brokerAddr, + tailConfig.tlsCACert, + tailConfig.tlsCert, + tailConfig.tlsKey, + ) } func tailRun(cmd *cobra.Command, args []string) error { @@ -146,19 +157,20 @@ func tailRun(cmd *cobra.Command, args []string) error { } adminClient, clientErr = clusterConfig.NewAdminClient(ctx, nil, true) } else if tailConfig.brokerAddr != "" { - useTLS := (resetOffsetsConfig.brokerCACert != "" || - resetOffsetsConfig.brokerCert != "" || - resetOffsetsConfig.brokerKey != "") - + useTLS := (tailConfig.tlsCACert != "" || + tailConfig.tlsCert != "" || + tailConfig.tlsKey != "") adminClient, clientErr = admin.NewBrokerAdminClient( ctx, admin.BrokerAdminClientConfig{ ConnectorConfig: admin.ConnectorConfig{ BrokerAddr: tailConfig.brokerAddr, UseTLS: useTLS, - CACertPath: tailConfig.brokerCACert, - CertPath: tailConfig.brokerCert, - KeyPath: tailConfig.brokerKey, + CACertPath: tailConfig.tlsCACert, + CertPath: tailConfig.tlsCert, + KeyPath: tailConfig.tlsKey, + ServerName: tailConfig.tlsServerName, + SkipVerify: tailConfig.tlsSkipVerify, }, ReadOnly: true, }, diff --git a/cmd/topicctl/subcmd/tester.go b/cmd/topicctl/subcmd/tester.go index faa45bfb..8ae17869 100644 --- a/cmd/topicctl/subcmd/tester.go +++ b/cmd/topicctl/subcmd/tester.go @@ -24,44 +24,29 @@ var testerCmd = &cobra.Command{ } type testerCmdConfig struct { - brokerAddr string - brokerCACert string - brokerCert string - brokerKey string - mode string - readConsumer string - topic string - writeRate int - zkAddr string + brokerAddr string + mode string + readConsumer string + tlsCACert string + tlsCert string + tlsKey string + tlsSkipVerify bool + tlsServerName string + topic string + writeRate int + zkAddr string } var testerConfig testerCmdConfig func init() { - testerCmd.Flags().StringVar( + testerCmd.Flags().StringVarP( &testerConfig.brokerAddr, "broker-addr", + "b", "", "Broker address", ) - testerCmd.Flags().StringVar( - &testerConfig.brokerCACert, - "broker-ca-cert", - "", - "Path to broker client CA cert PEM file if using TLS", - ) - testerCmd.Flags().StringVar( - &testerConfig.brokerCert, - "broker-cert", - "", - "Path to broker client cert PEM file if using TLS", - ) - testerCmd.Flags().StringVar( - &testerConfig.brokerKey, - "broker-key", - "", - "Path to broker client private key PEM file if using TLS", - ) testerCmd.Flags().StringVar( &testerConfig.mode, "mode", @@ -74,6 +59,36 @@ func init() { "test-consumer", "Consumer group ID for reads; if blank, no consumer group is set", ) + testerCmd.Flags().StringVar( + &testerConfig.tlsCACert, + "tls-ca-cert", + "", + "Path to client CA cert PEM file if using TLS", + ) + testerCmd.Flags().StringVar( + &testerConfig.tlsCert, + "tls-cert", + "", + "Path to client cert PEM file if using TLS", + ) + testerCmd.Flags().StringVar( + &testerConfig.tlsKey, + "tls-key", + "", + "Path to client private key PEM file if using TLS", + ) + testerCmd.Flags().StringVar( + &testerConfig.tlsServerName, + "tls-server-name", + "", + "Server name to use for TLS cert verification", + ) + testerCmd.Flags().BoolVar( + &testerConfig.tlsSkipVerify, + "tls-skip-verify", + false, + "Skip hostname verification when using TLS", + ) testerCmd.Flags().StringVar( &testerConfig.topic, "topic", @@ -86,9 +101,10 @@ func init() { 5, "Approximate number of messages to write per sec", ) - testerCmd.Flags().StringVar( + testerCmd.Flags().StringVarP( &testerConfig.zkAddr, "zk-addr", + "z", "localhost:2181", "Zookeeper address", ) @@ -246,16 +262,18 @@ func getConnector(ctx context.Context) (*admin.Connector, error) { } return adminClient.GetConnector(), nil } else { - useTLS := (testerConfig.brokerCACert != "" || - testerConfig.brokerCert != "" || - resetOffsetsConfig.brokerKey != "") + useTLS := (testerConfig.tlsCACert != "" || + testerConfig.tlsCert != "" || + testerConfig.tlsKey != "") return admin.NewConnector( admin.ConnectorConfig{ BrokerAddr: testerConfig.brokerAddr, UseTLS: useTLS, - CACertPath: testerConfig.brokerCACert, - CertPath: testerConfig.brokerCert, - KeyPath: testerConfig.brokerKey, + CACertPath: testerConfig.tlsCACert, + CertPath: testerConfig.tlsCert, + KeyPath: testerConfig.tlsKey, + ServerName: testerConfig.tlsServerName, + SkipVerify: testerConfig.tlsSkipVerify, }, ) } diff --git a/pkg/admin/connector.go b/pkg/admin/connector.go index acd8b345..f99dfba6 100644 --- a/pkg/admin/connector.go +++ b/pkg/admin/connector.go @@ -8,6 +8,7 @@ import ( "time" "github.com/segmentio/kafka-go" + log "github.com/sirupsen/logrus" ) type ConnectorConfig struct { @@ -16,6 +17,8 @@ type ConnectorConfig struct { CertPath string KeyPath string CACertPath string + ServerName string + SkipVerify bool } type Connector struct { @@ -29,14 +32,18 @@ func NewConnector(config ConnectorConfig) (*Connector, error) { Config: config, } + var tlsConfig *tls.Config + if !config.UseTLS { connector.Dialer = kafka.DefaultDialer } else { + log.Debugf("Loading key pair from %s and %s", config.CertPath, config.KeyPath) cert, err := tls.LoadX509KeyPair(config.CertPath, config.KeyPath) if err != nil { return nil, err } + log.Debugf("Adding CA certs from %s", config.CACertPath) caCertPool := x509.NewCertPool() caCertContents, err := ioutil.ReadFile(config.CACertPath) if err != nil { @@ -47,9 +54,11 @@ func NewConnector(config ConnectorConfig) (*Connector, error) { return nil, fmt.Errorf("Could not append CA certs from %s", config.CACertPath) } - tlsConfig := &tls.Config{ - Certificates: []tls.Certificate{cert}, - RootCAs: caCertPool, + tlsConfig = &tls.Config{ + Certificates: []tls.Certificate{cert}, + RootCAs: caCertPool, + InsecureSkipVerify: config.SkipVerify, + ServerName: config.ServerName, } connector.Dialer = &kafka.Dialer{ Timeout: 10 * time.Second, @@ -61,6 +70,7 @@ func NewConnector(config ConnectorConfig) (*Connector, error) { Addr: kafka.TCP(config.BrokerAddr), Transport: &kafka.Transport{ Dial: connector.Dialer.DialFunc, + TLS: tlsConfig, }, } diff --git a/pkg/config/cluster.go b/pkg/config/cluster.go index bf002cc8..3b8629cf 100644 --- a/pkg/config/cluster.go +++ b/pkg/config/cluster.go @@ -89,6 +89,8 @@ type BrokerAuth struct { CACertPath string `json:"caCertPath"` CertPath string `json:"certPath"` KeyPath string `json:"keyPath"` + ServerName string `json:"serverName"` + SkipVerify bool `json:"skipVerify"` } // Validate evaluates whether the cluster config is valid. @@ -157,6 +159,12 @@ func (c ClusterConfig) NewAdminClient( admin.BrokerAdminClientConfig{ ConnectorConfig: admin.ConnectorConfig{ BrokerAddr: c.Spec.BootstrapAddrs[0], + UseTLS: c.Spec.BrokerAuth.UseTLS, + CACertPath: c.Spec.BrokerAuth.CACertPath, + CertPath: c.Spec.BrokerAuth.CertPath, + KeyPath: c.Spec.BrokerAuth.KeyPath, + ServerName: c.Spec.BrokerAuth.ServerName, + SkipVerify: c.Spec.BrokerAuth.SkipVerify, }, ReadOnly: readOnly, }, From bf3539ccfb72a6359a67d73c297a5e0047171994 Mon Sep 17 00:00:00 2001 From: Benjamin Yolken Date: Tue, 24 Nov 2020 19:03:49 -0800 Subject: [PATCH 06/19] Fix cluster paths --- docker-compose-tls.yml | 39 ++++++++++++++++++++++++ examples/local-cluster/cluster.yaml | 1 - examples/tls/certs/ca.crt | 19 ++++++++++++ examples/tls/certs/ca.key | 30 ++++++++++++++++++ examples/tls/certs/client.crt | 20 ++++++++++++ examples/tls/certs/client.key | 29 ++++++++++++++++++ examples/tls/certs/kafka.keystore.jks | Bin 0 -> 4413 bytes examples/tls/certs/kafka.truststore.jks | Bin 0 -> 1106 bytes examples/tls/cluster.yaml | 18 +++++++++++ examples/tls/topics/topic-default.yaml | 17 +++++++++++ pkg/config/cluster.go | 32 +++++++++++++------ pkg/config/load.go | 15 ++++++++- 12 files changed, 208 insertions(+), 12 deletions(-) create mode 100644 docker-compose-tls.yml create mode 100644 examples/tls/certs/ca.crt create mode 100644 examples/tls/certs/ca.key create mode 100644 examples/tls/certs/client.crt create mode 100644 examples/tls/certs/client.key create mode 100644 examples/tls/certs/kafka.keystore.jks create mode 100644 examples/tls/certs/kafka.truststore.jks create mode 100644 examples/tls/cluster.yaml create mode 100644 examples/tls/topics/topic-default.yaml diff --git a/docker-compose-tls.yml b/docker-compose-tls.yml new file mode 100644 index 00000000..beeb85f4 --- /dev/null +++ b/docker-compose-tls.yml @@ -0,0 +1,39 @@ +# This config sets up a simple, single-node cluster that's equipped to use TLS. +# See examples/tls for the associated cluster configs and certs. +# +# To verify that TLS is working properly, try something like: +# +# openssl s_client -debug -connect localhost:9093 -CAfile ca.crt \ +# -cert server.crt -key server.key +version: '2' + +services: + zookeeper: + image: "wurstmeister/zookeeper:latest" + ports: + - "2181:2181" + + kafka: + image: wurstmeister/kafka:2.12-2.4.1 + ports: + - "9092:9092" + - "9093:9093" + environment: + KAFKA_ADVERTISED_HOST_NAME: 127.0.0.1 + KAFKA_BROKER_RACK: zone1 + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_LISTENERS: PLAINTEXT://:9092,SSL://:9093 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://127.0.0.1:9092,SSL://127.0.0.1:9093 + KAFKA_BROKER_ID: 1 + KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true' + KAFKA_SSL_KEYSTORE_LOCATION: '/certs/kafka.keystore.jks' + KAFKA_SSL_KEYSTORE_PASSWORD: 'test123' + KAFKA_SSL_KEY_PASSWORD: 'test123' + KAFKA_SSL_TRUSTSTORE_LOCATION: '/certs/kafka.truststore.jks' + KAFKA_SSL_TRUSTSTORE_PASSWORD: 'test123' + KAFKA_SSL_CLIENT_AUTH: 'none' + KAFKA_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: '' + KAFKA_SECURITY_INTER_BROKER_PROTOCOL: 'SSL' + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - ./examples/tls/certs:/certs diff --git a/examples/local-cluster/cluster.yaml b/examples/local-cluster/cluster.yaml index d6d78d6e..812a6d57 100644 --- a/examples/local-cluster/cluster.yaml +++ b/examples/local-cluster/cluster.yaml @@ -12,4 +12,3 @@ spec: zkAddrs: - localhost:2181 zkLockPath: /topicctl/locks - useBrokerAdmin: true diff --git a/examples/tls/certs/ca.crt b/examples/tls/certs/ca.crt new file mode 100644 index 00000000..563c9e0d --- /dev/null +++ b/examples/tls/certs/ca.crt @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDKDCCAhACCQCxcVKD5FwhXjANBgkqhkiG9w0BAQsFADBWMQswCQYDVQQGEwJV +UzELMAkGA1UECAwCQ0ExCzAJBgNVBAcMAlNGMRAwDgYDVQQKDAdTZWdtZW50MRsw +GQYDVQQDDBJ1c2VyLnNlZ21lbnQubG9jYWwwHhcNMjAxMTI0MDI1ODIyWhcNMzAx +MTIyMDI1ODIyWjBWMQswCQYDVQQGEwJVUzELMAkGA1UECAwCQ0ExCzAJBgNVBAcM +AlNGMRAwDgYDVQQKDAdTZWdtZW50MRswGQYDVQQDDBJ1c2VyLnNlZ21lbnQubG9j +YWwwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC4ayiwKN/iS06RJwOL +Bey75INaq92gsPT8yOI2/u43hqO7wiC6AHnf1nais/4P1zUuS6WZeA5rUsJPzhKC +N6fYFNkEMA5ui7LoEjJqD6o4Bw1cxWvQ/+Y9GwDOdK6T/q/ZSu9W7TB/Lgi0dT3C +SNfr/KnBDwsSUjEV7WP84qfbikUInPx7doFTm/pa6J44LvvdLH3qBqdiWdjKP/K3 +9mzADbNAQeLReGBEouaVBdIccDVoNfcG1/f+DDcupCjzFMUM8Hu5991a7Z7f1A1T +FalZmbk+THiP/4lliQgKfvhfVryVmq8YVsPPEiyy9biF/qtrJKFMbxsNn2eqP9ED +ZRCnAgMBAAEwDQYJKoZIhvcNAQELBQADggEBAHnHNiOQ02HfJXcLVAAJFMrfe2a8 +cfoD58oB0xuXEjHAHqR4iwp01S8fssNw82RdIxymPD8nfOkyRnd5g+gYgTOJsTL9 +ed3UGIH8dSel367z5dxYL3fVGZttqUutDH60TtUVc+qS1W+Vt6pEexETDo5PYj/D +Ho4f3YoORg+w4V0ZUCX3mgNYxyJULuPLBeYn4bXnplCcxG9rZgxMtIOhMnLlYdtn +nJbiruRmReJk+Ifkx//38YkYFjxWuztdWmF3nVotyUNCAV0cb5E/0w8nLwrR8JZr +ychNKuBcy92RJ7XYFMytI4PqisXiQ/VP5R0tpjVxLMAonF7hWj7n+5LSM8I= +-----END CERTIFICATE----- diff --git a/examples/tls/certs/ca.key b/examples/tls/certs/ca.key new file mode 100644 index 00000000..93b51fb1 --- /dev/null +++ b/examples/tls/certs/ca.key @@ -0,0 +1,30 @@ +-----BEGIN ENCRYPTED PRIVATE KEY----- +MIIFHzBJBgkqhkiG9w0BBQ0wPDAbBgkqhkiG9w0BBQwwDgQIDqzKTLtzeZcCAggA +MB0GCWCGSAFlAwQBKgQQkYvJZrDaFnldNi2Wr53f9wSCBND9mXn1ii3nsPieqy07 +7Gl3iZ0N18jRJfqsbAcbj31Vktcp+dhMLFbz51iFBTqq7xESUFrZEaGSpwqGrLV7 +NrvkDgq2khqQGh1dlCbYYyBfq2vyxgpeISxrHCNUtfsBpT6x/r8lLjxrjD2m/PjR +J80ZSfV1KpaSEP4HPR01mC3EwF+wdwK3NYJWkk0PcKKZVhXVFg6v3x6RJhnU7jW1 +aKnrT/GWdrhSqTZBvrrW4+x0wr/Iygw/MxuO70k4fpYWfrcLIEs9Ia5RVpFS1lXz ++DRBLZnjsyv7BXSeDRBEWcx1pb5CGuwgQ4yRx2xK2lRReX6lIE53YLD/1nK8xSzS +5JorKefkwsHhqgfRrzZu6aLMe7f7PJmrfQ9L4dLc27MC7HIrBnA2RZLfHkp9hGOP +73H5U6s6OhvFC81ECh3+ejHVk+Zb6Rx6Xi/tOPoPNLNiEXaLP9jpCo1uLqnvRdpG +PyjpuP0goOb6u/1HU7Nfxqyxf4LVPb0DTwnooXPveJr9czhuboYjhCk48cheTYy1 +kiCym2aKQv62ZxiCVVcHuoCXTbRZKAUtfYG3CQBn3a2Y0I2r1gODeMokpNHuSA9L +htuOuXx5SYKWxSeGsT0FePddHOBWRTOw3OB9fj95/vHVs7Tkiw6Uj9Sr/Q7rtm2O +KuLBw9QvZKSzpJlZ3nKaylWSXJp6zOx32UsrK+XQhL3UCbS2BkZxA5s1VVIg9p4+ +c0gqflIDvoBHM4elLZ/X2AV0lsdfNvyPx+wmI4nW5sazPbeIXY/MzOv7KwSeZkea +phGufA0ioEQUlyJ+P5iDOsaD1uGH7rdyIOGWnaP6jiGAkQ6IoorLSMvzdCHysuZp +ZziOfLL8ydhxpkma4VK1PYHFNlk72T/qeHDRr110osvMiE2DuMQ9RPbLR4hrrEDM +WUx4fgnpCqDDx2/1YScXioMY4BwEHliS7hZlv27ODWRO23YdQSB/e//yh9VBQnGS +dFw1dmtu1XDe+78l/YbuSzE3ppYO6VAMRiFgrWUubZgS5qmzAHn0i68ERnlK7j9s +jQYGltuHetrMnW7aBsV+TU3QsOqWrF9hL1geydwKezXGYkEURY69zLrlVkNxhFtQ +VJNRo9YKiFrdE/94qrhjyNZOwmq/3vs2+XEUGhCCp+Pfxtg8jJNSuEoXKWaaTwtS +fOae6wtbeydQG6iVYz7Mdm/Sc2xKRm3aVz4mkCxcfdyKmP8W0Yiw+6GniNR1Ryfu +gyr2LN44MaPH5fazYVPapocm1xK4HDVAHjFLzfzWbYde8fzf21LWWad5pwsClimL +4GfkFp0KGfajlcDMquJWn8ILU41unXo3Q/ikV3w8HEr6KydqRYXDctcm6WAx/8NP +8YXP9NEcq8FCKqjGlenjSwo1YfUJnmDEglpX8gCdYzEe0n6aEq2TfY8AsZ/rzls6 +BxRcnR7U3zR2maDr4gAYo7WR2pGe9WbgT+arwtBValmDI5C2sAu83MB6ubGA/lDR +M+DhLy69s4QeKI/X35bUVY3Qjm09Zxj4NOgVZwWV6mEAyMMLyKRF+NPnBdMSxQRe +u6Jpt4Ig1RUXMRNSv3u86W/PFmS7gAQEBMYL2rVrSeF8Vlb/uVPC31fUFDmFfQ3x +nPWzjMvCjivfzVGp5vo/Sd6OLw== +-----END ENCRYPTED PRIVATE KEY----- diff --git a/examples/tls/certs/client.crt b/examples/tls/certs/client.crt new file mode 100644 index 00000000..e6b1dace --- /dev/null +++ b/examples/tls/certs/client.crt @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDNzCCAh8CCQCBQTsa+CfoGTANBgkqhkiG9w0BAQUFADBWMQswCQYDVQQGEwJV +UzELMAkGA1UECAwCQ0ExCzAJBgNVBAcMAlNGMRAwDgYDVQQKDAdTZWdtZW50MRsw +GQYDVQQDDBJ1c2VyLnNlZ21lbnQubG9jYWwwHhcNMjAxMTI0MDI1OTQxWhcNMzAx +MTIyMDI1OTQxWjBlMQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExCzAJBgNVBAcT +AlNGMRAwDgYDVQQKEwdTZWdtZW50MRAwDgYDVQQLEwdTZWdtZW50MRgwFgYDVQQD +Ew9CZW5qYW1pbiBZb2xrZW4wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB +AQCdQR9sA4Rv7sPzBGO5FMTSq7ULxvCEIONE2zppv7DqOCmz6Lq2//7tzCeSNt7p +vh5HH9tpwIN5b8kwoppMKGOVPTmfdbTXXwsSP3JPZJnPSFdoElBh/qnhoKhF6tLD +um3fZFbZ66KintcG5/9PgluanOr4WuakW3YBs9SLwEY28ZvkeBcwfSZENFh74dr6 +alkfn+u+0zhyRVvbT6A0gr2zeRUb35UAS5Vlgc5zI65v+d7TT7OaIHkL8rnrsMft +DqBlPhTzlfGz64ipFgFQuTHuFWg5jenwKuLaDq2tMawRLaM0ZZCAu7Tmlmq429VD +CP1wEq1DKUWQ7lR1me9Cft+3AgMBAAEwDQYJKoZIhvcNAQEFBQADggEBAIntZGmd +1tSxOfgqGz3vQSqoYhC9tsEWbwGnez7c2WLJE3vR6pHB06iWqegf78DAWkz1ZI+d +1OPYfqPw63sOZYsHIoug5xR8QtU8G3NaT1H7Vc7GiIkU+PIn0V7DzBSqEZWuoTz3 +XUbH25O5ynMKyGR6zirRZLDL1lw1dDKeqbaUPt/QxuY1S6Pl+36C2DOBDWqJJWQJ +rtxJ4zFA+ZJEK1EKIJF7ufM0qfCCnKTnvo/4SLGItPrmp/xUFl5T4ises/uLqrY4 +fWbpV8hbuPo5f50AD6S1Iw7H1ZVKyHazBIYaC0QS8Vk4vp5I6J7OnuR+jhv0y0vr +Bd/jWLKcAv0MzjQ= +-----END CERTIFICATE----- diff --git a/examples/tls/certs/client.key b/examples/tls/certs/client.key new file mode 100644 index 00000000..d788fc63 --- /dev/null +++ b/examples/tls/certs/client.key @@ -0,0 +1,29 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCdQR9sA4Rv7sPz +BGO5FMTSq7ULxvCEIONE2zppv7DqOCmz6Lq2//7tzCeSNt7pvh5HH9tpwIN5b8kw +oppMKGOVPTmfdbTXXwsSP3JPZJnPSFdoElBh/qnhoKhF6tLDum3fZFbZ66KintcG +5/9PgluanOr4WuakW3YBs9SLwEY28ZvkeBcwfSZENFh74dr6alkfn+u+0zhyRVvb +T6A0gr2zeRUb35UAS5Vlgc5zI65v+d7TT7OaIHkL8rnrsMftDqBlPhTzlfGz64ip +FgFQuTHuFWg5jenwKuLaDq2tMawRLaM0ZZCAu7Tmlmq429VDCP1wEq1DKUWQ7lR1 +me9Cft+3AgMBAAECggEADqdR4UPWpIOQWOXw0P9hc+wyO723DejupKz1HYOSXdEL ++crXE1R5kfkzOsnILenccm5CiPE6jydejRyp2iztUqvY4cYbKvKdWn71DPbn6kvo +cTc7rFYJyI+q/pDqQPjvYiC8gyQVDKhWizs1LFiOZrL2plv6IBixv2jdhoRNRrNI +65N6Z9wHTNztcJyKDKT6Z4pgBDYSWzUIAGVfw2vhYwDavgiSTAOksGc15rmamc0+ +ZYaDpwfbFEI06jBIJEBzcfdp/0FtNQSTL0FDkXdQyBhc6IxNHIifL5dDvhMo0Aks +dcnlBf13DVskLe7lVxOPuXoljC6Us1WkWJ2VWJ6RYQKBgQDyvxqInnqVS2AK8t+X +q7w4yccv+2exPM5gs9Bht9C/e/OPb39bSAdjLOJOsTAkkBQfTyrawEqUGFsq5y1B +U4SljPJchXfL82pefXfIHCtqhv8ieJb2Tby50PQzeY2+pw8/0xgbVjTAr4JKRXO0 +hyMEOj6ElKFnjHlpqSuCl5hGsQKBgQCl1xX1uBimphJJ3Zo74UUop87GeFbWl4oL +pwqI3DOy5KMrbRtWqU3EI8BkjgjHJLF3d4evVAIoBG54fZFjHSPgq3163WUuU6ZW +p1gvHTWyTNbuu8IFqqhbE/vHUZMfX7ksuBooFOsY0Wqq0poVgDWriFS5SfYek43f +WAZiAvn25wKBgFAd3qYEmDS6AeLbMgye86pSfllJwnlutjaYYkg+ILlyMXq/s+ru +pPGImNCcDmWi3+FNgbldCcBDIaPRVNBgvkDdeggrTNSVbB/vjR8QnQu1rnM0Fa8J +DSbO3io3Dh9Eh/Xqt+Qd2Z9WzcuxjHSivV3h00xyuaqxZEkJOoEJg4qhAoGADOND +JJ5S5BiB0VW0V7Tw7/Dig8/0R6btJmyrx+j854kXGRfYiQqNLZHtsKLNEdTLKdKT +K8/mfv+hKiHv+3jXQe1xyeuMomYDxjYpBzhI5PtNtK3IrTIO9Uz/QwUW3thMhqoj +9jtx7bLQjEfji4o0IYlttByIUOX8n3+yt0kt7b8CgYEA0NMkFD39NBYv4Ec5fx7y +bNcObdI35cyw5xt2uSqC/2fcqLeD8FKlsS9b9JkkhYvEEe1z/3vMuO7erpQuh5j6 +P9gO/TcaZOk9DACoI2hEfUlW4SX2rLPbPB0ZG61PStjeyf1uy78jCVQYjM7N+WBE +IY2Y+bXLOjn24ooplcIlZsw= +-----END PRIVATE KEY----- + diff --git a/examples/tls/certs/kafka.keystore.jks b/examples/tls/certs/kafka.keystore.jks new file mode 100644 index 0000000000000000000000000000000000000000..c293b952333ac67ef8c2c44d53aac5d412a06229 GIT binary patch literal 4413 zcmY+EWmFW7w#8=#7@DEGLwXou2uVq4L8MDUkQho@1csFEMj8a9rMpWy|8z=smt5al z@7?$AhqLxsd!MsE{$NM~5fBgshQvPyV{!z)4!*$!q64#$_?xIm{MEnqA`FS<@E;MH zDJl}p@UN}=ceKFR|MP`|1q5XyQ6FGP)EgK(80&xSzvWb@#G9_7wjT;moW3gg>Gz>> zk)LTGI6x?X?l-7N)PWkg%$FwFk`b4E5@PmfgW-+N?0cFvxyDI!+#vyrZI-nbzQFxi zG`JFGk&aCPN(PyQi-WawNSt`RX6bB`vjXg~PWF4#0D)G>1L?M5lM z)C^uay|U=k>~KAUni|ecCE?bN+_89Jz5QqC0BjEQ-4Ff-fKfSqn$UU4hW)VSgfpA( zM*~GqNbMc+b@HzI>b?Wkx7LpYgNgsmSLEcj2zRxxL=R-+b^~JhkCdTLjBW(7oInuALChgzG?xFkE2Q(ZATml4N{J+t_e%MKKg_L~$_i?TUNrOj$)jjh+AKkz8{0DYuHfi*e4-v{8>Qw0 z$(!8j5se&y*=$hv0?3E$GB;&<^6?kl3?ZRP)>k{IPgnw4$LBr(#ZCUf+wqqMwVXUb zi}f6{Ka-NE@%ZLS3h}AB`Rj9BmI`MWJ&j>dcaPYtCMW(dHQXD8e?C zb6m$7lUSfaU6)_mD!(H&qId4C2vndPY@g`;PD6r4eU$fb3-_Ul%}@{{fD-|07?Q ztJeDZmnWr|(EgREbiBe;_}j_BxIkD-jE_cmwQfIQp9Fb}Pc<}Bu;r7LLa;Ra&7}IB z>Erw!yisA(jS5+LC-|L%1;*=vDlq+7uSBg#76o-uuAmUk&Iz|0gn+npF0hks`pRzm zwcnY(KA>maUP7`FjP&>?)_iBQE_Y=kv>`_*vO*;r>cPMNuyaXvAXIw*gjWsKXU&(# z;_3a;YHwZ^yN2PTktOe0Ev^u!MsPBcOBN@kyk^w&YDgeHTu=GwXv>wDdYZk`%svZ}GK(iW&V8*n` zwiq%%^Zw@6_{FrMBKu~&L5@^K>Tc=q$D5t(QL{LWMv-U8pYr763!#$hpJViSpKL=q ztpQEb_T~#KBUg%ryWAb~EItZ+7_Gs&4$>!Q8Qa@Pg{v#<5d?wa2d^eI37wc#iOoK`!zMC{At1$|C@@k&HXpowV`cg zp8al<%jKD%xyJ#U@G;fF7}uLz32!vR3_Qs#B})ulSB0Q=@IT7cPbAI-`XiA8s1djf zDMs8_<+DFbW{!-zcO3$~>gFGk5pSih)oJjI_`g7uR5H-4kA+-qboD->H*{-Bun^9E z6xM4HFmiT?NHTT|zB(`BXjY%DQ&dEfmXY1hsNiE^@*> z&|qNR6&f!aSk;exEr}A5Ryq;~k$J}!gT(0~`$md0;1SGWcWH#gM;|DNRgz+Ouu88x zNq*#u4Qn@rQ^lsPn55pXIc-0s_sZXa@^iv<-RRJPq{aR|LW)zdEuDg`Hd^MmpIOK9 zJND;WLNkvBM}5@|?v2zRTAOWRVqFaLL#!Hpdo9>kzdtxM^9&Nbps+U}1GVwZiH*y{ z=*ImDj)Dyu>0}1}tcH%$mD4rctGat@VcSILGOP9We2c7h)rBs8s{5sU(N)~3{l+Oy;p61B zHfY>GY<5TPX0WC1GyK?8ds(^_<&M0Ze#?k@9xi6n?Z*Kbi;gbKO%OSHOpfQ060t?~ zUqddC)J=E@i($8tj-HHu>Z_gtV=U7sCu=PKLj^l28*MCvF?cuwtmA*4b+W0%4UzR< z)mjYl@~OgkXVEhmB`{WVI2AfYJ(l%tP_R#T6rJ;V3#x~=P0$``_f{>UcoyXccSqaE zZx)qUBtr93^YtvYSfVmZ4{LUJ2-83mm7-Oa@gEgNBdwity7h)4Tw(||AjglpZf;-gbVp579@^3nLlN_Q@qT*8MW# z$4mULs$;agYdwv#c3zQk664cl`vj)L!d=LK31*XzEDB`n5C;oaFtoimSN&20@A_b- zIX6rCWsb8B89Dv;T-qgwq9qIJtz)QKT+yiimk1`u2$z;bjWlEkx&lO%cC)d7(&_v1 z)Y`rxxT>~}5!0>j*j(q7sL5Ziu)SV*YRw$vrJY3>pF%rH5u<8)q-Kz#fM^N(WJU6p zO(**m^KvZR5JwjDUThHKz#yZ+oybk{z3VS7BA19t6~Vd%K&4-`XrLTdpv7J!#$d*HyHx>{g~7O|u67E!t&LyL96twe~HE*Y-$qhT*v@DcN*e zEus&wl=n)60uo31wT?|i)8SyNiGzvvv}NH7xBUWwHvdQ1M5M{3)VeL_xG{FhYccWC zvKsX9Svt(*;-{U7=G`hRaxZtwH?^b?e8*9^s8Y#;>JbprbA@i}5G05Wrxedh7 zCJQRhu1z%5k{9FKI{T1XFVI##PW|t3Qjre3I(=WMn8fS?3>&peL|T6ly`P?-v+VIR+9LG zFE64UT6?V_RV;KKZhK^jYh7=C&rqp+0n2eRL{_9&E^82)X~eLMjk;EA!V~Dd*&~Nn zPPEHY(Inj(84tcHPD1TRy?eqR%U%q_zA=tPJ~m2UeK)?qr6&P=GXW92f1WSNZv2*~ z=@{_bR%Jr7w9#)I-pv2;CYA}GAQofBqi9B@Mdof=MtmV=BT!!{u-=Qf!EpTDB7doJ zX|m=B({JvB=+@gRMK;Ryqk}*8VhBlx({1#kZA!X=4LH#uo_7!Kuq*!ADQXq)V@bT!9kJtF0B_h>9rViD)4N|H5LkfFf zmF2pKfLG~)3j8HRvygx_A>sKuSfyCGLcXu$%@vcSVTA+236ZY1&1q|JwKgv>#Cpay z-*6fTc1SPO?r5g%?3Nt|>SH(xCf@WgTs*dW8|bgY=^=Tchr`)N2$sNsAxhgy@q^qa zQlBghQ<$`K=m+XES5n&tZO-DA@s92Wbnm`$tBkif_#d|1{wWzIk=EnQpXsi}t@6vlqDgpxm{iR}=xT(`HMgQwjw3V&&2kQ9&k)aM zaLeZ<6^g=TWtmq>$e8qH#%fFD*7}vOCb3Ktf_N5syYP7EFS~;?mXLp#OFW)bB~*UH zeBQwwy}kthX+x>d(p;%y2YqiGW;>;XgS@JxL9jdB_&77NoT<|?B)Uo&arj>c6Kk&h za;U~hwq%a}($DP|VZ#UV;VHXCXjYHJpCM=srTO?NeK*r|O#Lkr?p_ocb=3%_5Zlyi z5-VX=bt?yO$NC|M`4| zcO|ZB;#GDn9tZhGR>`hjl?Zx|XtW-GZ%7pLZt<^Kkd96qC%L3Nlz;HLo5o)vgKKCO+^w@u=9#zNE-3v7}?UrN!v& z(9nCCV@I7BR`x$cC_S%!$PvYORZu(SZ4TEJ?1?`YnI`p@uRvtsY)`z`LK<>&r^Y^9 z5oTI=-cqs!;H@4mCS>AKeh7JkT|hE#ZjKq1bSKZpAnWS%6mdAzX&Fs6&+E}cne8}q zf`HOlG5tMsB2=#10pLN;1y(b>gAIR>V1!HIb)j5T`dW4IyGT1w@GkKqU^27br}_aMQm{SGgR**o2zx itRA2e$K`$P_5So?@2oNd3xA`tc5Wao4}<~?&i)rX06Clh literal 0 HcmV?d00001 diff --git a/examples/tls/certs/kafka.truststore.jks b/examples/tls/certs/kafka.truststore.jks new file mode 100644 index 0000000000000000000000000000000000000000..432c94af4b743b379621671b96fb15b00f4130e6 GIT binary patch literal 1106 zcmV-Y1g-lpf&@+i0Ru3C1P2BQDuzgg_YDCD0ic2d_ymFj^e}=0@Gyb{>;?%chDe6@ z4FLxRpn?P4FoFZw0s#Opf&eZUuZdag98~;O^C`X6)4PuavvqwFX6~)$%2&D zZHjsM7ZDPmjdKTYOTOlEcrKBN-@qp1cv|bk1dGm+7vi8RAG2zPJ>N3JwYIK+2wWkr zd-F=>$U&LnDvBzyOilpOHc1dN)mACHV?EhB$C07d{$*d1l9D3Pxex3Lw^e9x!f<9qkA0-y2$*Q2e;7#NTL~!5}9d3A@rn7Q9K^WDpr8|?zEG=fr z!vI}Z=``i{XnHPEO=b-WU%G)BS)QcF2MkKt;CQYJq&lyWCUenU%Ts0pDpI`y6X>5@ zuYTL0c7{^w&jV$RFYcI>T0TZ)XT6SJ~EU1gHX z@h3dX%3b=jOkGj-x)J4Lz$IW6vso|`Wi&}7#RL7dx^TPt(isM{bHS*DI&3m1nbLOq3G z03cKMT3ipmJPm~~2oZ^66z5CSN>yE>#d22LIN%&?V`*`@O!I>LDTTeOu1S5s1b>yS z5;x)gY?Y4>#Y&K!Qn%Ze=e*iAS Date: Tue, 24 Nov 2020 19:35:06 -0800 Subject: [PATCH 07/19] Update README --- README.md | 51 +++++++++++++++++++++++++-------------- examples/tls/cluster.yaml | 1 - pkg/config/cluster.go | 18 ++++++-------- 3 files changed, 40 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 8275ccc9..e70a7af6 100644 --- a/README.md +++ b/README.md @@ -25,9 +25,9 @@ more details. ## Roadmap -We are planning on making some changes to (optionally) remove the ZK dependency and also to support -some additional security features like TLS. See -[this page](https://github.com/segmentio/topicctl/wiki/v1-Plan) for the current plan. +We are in the process of making some changes to (optionally) remove the ZK dependency and also to +support some additional security features like TLS. See +[this page](https://github.com/segmentio/topicctl/wiki/v1-Plan) for the current plan and status. ## Getting started @@ -205,8 +205,9 @@ There are two patterns for specifying a target cluster in the `topicctl` subcomm 1. `--cluster-config=[path]`, where the refererenced path is a cluster configuration in the format expected by the `apply` command described above *or* 2. `--zk-addr=[zookeeper address]` and `--zk-prefix=[optional prefix for cluster in zookeeper]` +3. `--broker-addr=[broker address]` -All subcommands support the `cluster-config` pattern. The second is also supported +All subcommands support the `cluster-config` pattern. The last two are also supported by the `get`, `repl`, `reset-offsets`, and `tail` subcommands since these can be run independently of an `apply` workflow. @@ -376,8 +377,19 @@ the process should continue from where it left off. ## Cluster access details -Most `topicctl` functionality interacts with the cluster through ZooKeeper. Currently, only -the following depend on broker APIs: +### zk vs. broker APIs + +`topicctl` can interact with a cluster through either ZooKeeper or by hitting broker APIs +directly. + +Broker APIs are used exclusively if the tool is run with either: + +1. `--broker-addr` *or* +2. `--cluster-config` and the cluster config doesn't specify any ZK addresses + +In all other cases, i.e. if `--zk-addr` is specified or the cluster config has ZK addresses, then +ZooKeeper will be used for most interactions. A few operations that are not possible via ZK +will still use broker APIs, including: 1. Group-related `get` commands: `get groups`, `get lags`, `get members` 2. `get offsets` @@ -385,21 +397,24 @@ the following depend on broker APIs: 4. `tail` 5. `apply` with topic creation -In the future, we may shift more functionality away from ZooKeeper, at least for newer cluster -versions; see the "Feature roadmap" section below for more details. +Note that the broker-only mode is only possible with newer Kafka versions (>= 2 for read-only +operations, >= 2.4 for applies). + +### TLS + +TLS is supported when running `topicctl` in the exclusive broker API mode. To use this, you'll +need the following, each in PEM format: -## Feature roadmap +1. Client certificate +2. Client private key +3. CA certificate(s) -The following are in the medium-term roadmap: +These can be extracted from Java keystores (like the ones used by the brokers) using the `keytool` +command. -1. **Use broker APIs exclusively for newer cluster versions:** This is needed for a - [future world](https://www.confluent.io/blog/removing-zookeeper-dependency-in-kafka/) - where Kafka doesn't use ZooKeeper at all. Even before that happens, though, doing everything - through broker APIs simplifies the configuration and is also needed to run `topicctl` in - environments where users aren't given direct ZK access. -2. **Support TLS for communication with cluster:** This is fairly straightforward assuming - that (1) is done. It allows `topicctl` to be run in environments that don't permit insecure - cluster access. +You can then specify the paths to these via either the `--tls-*` args on the command line or +via the `clientAuth` section in a cluster config spec. See [this config](examples/tls/cluster.yaml) +for an example of the latter. ## Development diff --git a/examples/tls/cluster.yaml b/examples/tls/cluster.yaml index cf1a950a..71864af0 100644 --- a/examples/tls/cluster.yaml +++ b/examples/tls/cluster.yaml @@ -9,7 +9,6 @@ spec: versionMajor: v2 bootstrapAddrs: - localhost:9093 - useBrokerAdmin: true clientAuth: useTLS: true caCertPath: certs/ca.crt diff --git a/pkg/config/cluster.go b/pkg/config/cluster.go index 0b67475e..3c56df21 100644 --- a/pkg/config/cluster.go +++ b/pkg/config/cluster.go @@ -10,6 +10,7 @@ import ( "github.com/aws/aws-sdk-go/aws/session" "github.com/hashicorp/go-multierror" "github.com/segmentio/topicctl/pkg/admin" + log "github.com/sirupsen/logrus" ) // KafkaVersionMajor is a string type for storing Kafka versions. @@ -50,7 +51,7 @@ type ClusterSpec struct { BootstrapAddrs []string `json:"bootstrapAddrs"` // ZKAddrs is a list of one or more zookeeper addresses. These can use IPs - // or DNS names. + // or DNS names. If these are omitted, then the tool will use broker APIs exclusively. ZKAddrs []string `json:"zkAddrs"` // ZKPrefix is the prefix under which all zk nodes for the cluster are stored. If blank, @@ -79,10 +80,6 @@ type ClusterSpec struct { // limited by. If unset, no retention drop limiting will be applied. DefaultRetentionDropStepDurationStr string `json:"defaultRetentionDropStepDuration"` - // UseBrokerAdmin indicates whether we should use a broker-api-based admin (if true) or - // the old, zk-based admin (if false). - UseBrokerAdmin bool `json:"useBrokerAdmin"` - // ClientAuth stores how we should authenticate broker connections, if appropriate. Only // applies if using the broker admin. ClientAuth ClientAuth `json:"clientAuth"` @@ -117,9 +114,6 @@ func (c ClusterConfig) Validate() error { errors.New("At least one bootstrap broker address must be set"), ) } - if len(c.Spec.ZKAddrs) == 0 && !c.Spec.UseBrokerAdmin { - err = multierror.Append(err, errors.New("At least one zookeeper address must be set")) - } if c.Spec.VersionMajor != KafkaVersionMajor010 && c.Spec.VersionMajor != KafkaVersionMajor2 { multierror.Append(err, errors.New("MajorVersion must be v0.10 or v2")) @@ -133,10 +127,10 @@ func (c ClusterConfig) Validate() error { ) } - if c.Spec.ClientAuth.UseTLS && !c.Spec.UseBrokerAdmin { + if c.Spec.ClientAuth.UseTLS && len(c.Spec.ZKAddrs) > 0 { err = multierror.Append( err, - errors.New("TLS not supported unless also using broker admin"), + errors.New("TLS not supported with zk access mode; omit zk addresses to fix"), ) } @@ -157,7 +151,8 @@ func (c ClusterConfig) NewAdminClient( sess *session.Session, readOnly bool, ) (admin.Client, error) { - if c.Spec.UseBrokerAdmin { + if len(c.Spec.ZKAddrs) == 0 { + log.Debug("No ZK addresses provided, using broker admin client") return admin.NewBrokerAdminClient( ctx, admin.BrokerAdminClientConfig{ @@ -174,6 +169,7 @@ func (c ClusterConfig) NewAdminClient( }, ) } else { + log.Debug("ZK addresses provided, using zk admin client") return admin.NewZKAdminClient( ctx, admin.ZKAdminClientConfig{ From d6da49b7605f0aec75490038c8a30d982183fe72 Mon Sep 17 00:00:00 2001 From: Benjamin Yolken Date: Tue, 24 Nov 2020 21:52:22 -0800 Subject: [PATCH 08/19] Update name of tls enabled parameter --- cmd/topicctl/subcmd/get.go | 12 ++++++++++-- cmd/topicctl/subcmd/repl.go | 12 ++++++++++-- cmd/topicctl/subcmd/reset.go | 12 ++++++++++-- cmd/topicctl/subcmd/tail.go | 12 ++++++++++-- cmd/topicctl/subcmd/tester.go | 12 ++++++++++-- pkg/admin/connector.go | 37 +++++++++++++++++++++-------------- pkg/config/cluster.go | 6 +++--- 7 files changed, 75 insertions(+), 28 deletions(-) diff --git a/cmd/topicctl/subcmd/get.go b/cmd/topicctl/subcmd/get.go index b0efa02c..f54072a1 100644 --- a/cmd/topicctl/subcmd/get.go +++ b/cmd/topicctl/subcmd/get.go @@ -37,6 +37,7 @@ type getCmdConfig struct { full bool tlsCACert string tlsCert string + tlsEnabled bool tlsKey string tlsSkipVerify bool tlsServerName string @@ -78,6 +79,12 @@ func init() { "", "Path to client cert PEM file if using TLS", ) + getCmd.Flags().BoolVar( + &getConfig.tlsEnabled, + "tls-enabled", + false, + "Use TLS for communication with brokers", + ) getCmd.Flags().StringVar( &getConfig.tlsKey, "tls-key", @@ -139,7 +146,8 @@ func getRun(cmd *cobra.Command, args []string) error { } adminClient, clientErr = clusterConfig.NewAdminClient(ctx, sess, true) } else if getConfig.brokerAddr != "" { - useTLS := (getConfig.tlsCACert != "" || + tlsEnabled := (getConfig.tlsEnabled || + getConfig.tlsCACert != "" || getConfig.tlsCert != "" || getConfig.tlsKey != "") adminClient, clientErr = admin.NewBrokerAdminClient( @@ -147,7 +155,7 @@ func getRun(cmd *cobra.Command, args []string) error { admin.BrokerAdminClientConfig{ ConnectorConfig: admin.ConnectorConfig{ BrokerAddr: getConfig.brokerAddr, - UseTLS: useTLS, + TLSEnabled: tlsEnabled, CACertPath: getConfig.tlsCACert, CertPath: getConfig.tlsCert, KeyPath: getConfig.tlsKey, diff --git a/cmd/topicctl/subcmd/repl.go b/cmd/topicctl/subcmd/repl.go index 19bbf2ad..2ed03e80 100644 --- a/cmd/topicctl/subcmd/repl.go +++ b/cmd/topicctl/subcmd/repl.go @@ -23,6 +23,7 @@ type replCmdConfig struct { clusterConfig string tlsCACert string tlsCert string + tlsEnabled bool tlsKey string tlsSkipVerify bool tlsServerName string @@ -58,6 +59,12 @@ func init() { "", "Path to client cert PEM file if using TLS", ) + replCmd.Flags().BoolVar( + &replConfig.tlsEnabled, + "tls-enabled", + false, + "Use TLS for communication with brokers", + ) replCmd.Flags().StringVar( &replConfig.tlsKey, "tls-key", @@ -119,7 +126,8 @@ func replRun(cmd *cobra.Command, args []string) error { } adminClient, clientErr = clusterConfig.NewAdminClient(ctx, sess, true) } else if replConfig.brokerAddr != "" { - useTLS := (replConfig.tlsCACert != "" || + tlsEnabled := (replConfig.tlsEnabled || + replConfig.tlsCACert != "" || replConfig.tlsCert != "" || replConfig.tlsKey != "") adminClient, clientErr = admin.NewBrokerAdminClient( @@ -127,7 +135,7 @@ func replRun(cmd *cobra.Command, args []string) error { admin.BrokerAdminClientConfig{ ConnectorConfig: admin.ConnectorConfig{ BrokerAddr: replConfig.brokerAddr, - UseTLS: useTLS, + TLSEnabled: tlsEnabled, CACertPath: replConfig.tlsCACert, CertPath: replConfig.tlsCert, KeyPath: replConfig.tlsKey, diff --git a/cmd/topicctl/subcmd/reset.go b/cmd/topicctl/subcmd/reset.go index cc898e0b..b3a1892e 100644 --- a/cmd/topicctl/subcmd/reset.go +++ b/cmd/topicctl/subcmd/reset.go @@ -30,6 +30,7 @@ type resetOffsetsCmdConfig struct { partitions []int tlsCACert string tlsCert string + tlsEnabled bool tlsKey string tlsSkipVerify bool tlsServerName string @@ -77,6 +78,12 @@ func init() { "", "Path to client cert PEM file if using TLS", ) + resetOffsetsCmd.Flags().BoolVar( + &resetOffsetsConfig.tlsEnabled, + "tls-enabled", + false, + "Use TLS for communication with brokers", + ) resetOffsetsCmd.Flags().StringVar( &resetOffsetsConfig.tlsKey, "tls-key", @@ -138,7 +145,8 @@ func resetOffsetsRun(cmd *cobra.Command, args []string) error { } adminClient, clientErr = clusterConfig.NewAdminClient(ctx, nil, false) } else if resetOffsetsConfig.brokerAddr != "" { - useTLS := (resetOffsetsConfig.tlsCACert != "" || + tlsEnabled := (resetOffsetsConfig.tlsEnabled || + resetOffsetsConfig.tlsCACert != "" || resetOffsetsConfig.tlsCert != "" || resetOffsetsConfig.tlsKey != "") adminClient, clientErr = admin.NewBrokerAdminClient( @@ -146,7 +154,7 @@ func resetOffsetsRun(cmd *cobra.Command, args []string) error { admin.BrokerAdminClientConfig{ ConnectorConfig: admin.ConnectorConfig{ BrokerAddr: resetOffsetsConfig.brokerAddr, - UseTLS: useTLS, + TLSEnabled: tlsEnabled, CACertPath: resetOffsetsConfig.tlsCACert, CertPath: resetOffsetsConfig.tlsCert, KeyPath: resetOffsetsConfig.tlsKey, diff --git a/cmd/topicctl/subcmd/tail.go b/cmd/topicctl/subcmd/tail.go index a3219909..ecd19f5b 100644 --- a/cmd/topicctl/subcmd/tail.go +++ b/cmd/topicctl/subcmd/tail.go @@ -31,6 +31,7 @@ type tailCmdConfig struct { raw bool tlsCACert string tlsCert string + tlsEnabled bool tlsKey string tlsSkipVerify bool tlsServerName string @@ -84,6 +85,12 @@ func init() { "", "Path to client cert PEM file if using TLS", ) + tailCmd.Flags().BoolVar( + &tailConfig.tlsEnabled, + "tls-enabled", + false, + "Use TLS for communication with brokers", + ) tailCmd.Flags().StringVar( &tailConfig.tlsKey, "tls-key", @@ -157,7 +164,8 @@ func tailRun(cmd *cobra.Command, args []string) error { } adminClient, clientErr = clusterConfig.NewAdminClient(ctx, nil, true) } else if tailConfig.brokerAddr != "" { - useTLS := (tailConfig.tlsCACert != "" || + tlsEnabled := (tailConfig.tlsEnabled || + tailConfig.tlsCACert != "" || tailConfig.tlsCert != "" || tailConfig.tlsKey != "") adminClient, clientErr = admin.NewBrokerAdminClient( @@ -165,7 +173,7 @@ func tailRun(cmd *cobra.Command, args []string) error { admin.BrokerAdminClientConfig{ ConnectorConfig: admin.ConnectorConfig{ BrokerAddr: tailConfig.brokerAddr, - UseTLS: useTLS, + TLSEnabled: tlsEnabled, CACertPath: tailConfig.tlsCACert, CertPath: tailConfig.tlsCert, KeyPath: tailConfig.tlsKey, diff --git a/cmd/topicctl/subcmd/tester.go b/cmd/topicctl/subcmd/tester.go index 8ae17869..7c944d11 100644 --- a/cmd/topicctl/subcmd/tester.go +++ b/cmd/topicctl/subcmd/tester.go @@ -29,6 +29,7 @@ type testerCmdConfig struct { readConsumer string tlsCACert string tlsCert string + tlsEnabled bool tlsKey string tlsSkipVerify bool tlsServerName string @@ -71,6 +72,12 @@ func init() { "", "Path to client cert PEM file if using TLS", ) + testerCmd.Flags().BoolVar( + &testerConfig.tlsEnabled, + "tls-enabled", + false, + "Use TLS for communication with brokers", + ) testerCmd.Flags().StringVar( &testerConfig.tlsKey, "tls-key", @@ -262,13 +269,14 @@ func getConnector(ctx context.Context) (*admin.Connector, error) { } return adminClient.GetConnector(), nil } else { - useTLS := (testerConfig.tlsCACert != "" || + tlsEnabled := (testerConfig.tlsEnabled || + testerConfig.tlsCACert != "" || testerConfig.tlsCert != "" || testerConfig.tlsKey != "") return admin.NewConnector( admin.ConnectorConfig{ BrokerAddr: testerConfig.brokerAddr, - UseTLS: useTLS, + TLSEnabled: tlsEnabled, CACertPath: testerConfig.tlsCACert, CertPath: testerConfig.tlsCert, KeyPath: testerConfig.tlsKey, diff --git a/pkg/admin/connector.go b/pkg/admin/connector.go index f99dfba6..f7b7b844 100644 --- a/pkg/admin/connector.go +++ b/pkg/admin/connector.go @@ -13,7 +13,7 @@ import ( type ConnectorConfig struct { BrokerAddr string - UseTLS bool + TLSEnabled bool CertPath string KeyPath string CACertPath string @@ -34,28 +34,35 @@ func NewConnector(config ConnectorConfig) (*Connector, error) { var tlsConfig *tls.Config - if !config.UseTLS { + if !config.TLSEnabled { connector.Dialer = kafka.DefaultDialer } else { - log.Debugf("Loading key pair from %s and %s", config.CertPath, config.KeyPath) - cert, err := tls.LoadX509KeyPair(config.CertPath, config.KeyPath) - if err != nil { - return nil, err - } + var certs []tls.Certificate + var caCertPool *x509.CertPool - log.Debugf("Adding CA certs from %s", config.CACertPath) - caCertPool := x509.NewCertPool() - caCertContents, err := ioutil.ReadFile(config.CACertPath) - if err != nil { - return nil, err + if config.CertPath != "" && config.KeyPath != "" { + log.Debugf("Loading key pair from %s and %s", config.CertPath, config.KeyPath) + cert, err := tls.LoadX509KeyPair(config.CertPath, config.KeyPath) + if err != nil { + return nil, err + } + certs = append(certs, cert) } - if ok := caCertPool.AppendCertsFromPEM(caCertContents); !ok { - return nil, fmt.Errorf("Could not append CA certs from %s", config.CACertPath) + if config.CACertPath != "" { + log.Debugf("Adding CA certs from %s", config.CACertPath) + caCertPool = x509.NewCertPool() + caCertContents, err := ioutil.ReadFile(config.CACertPath) + if err != nil { + return nil, err + } + if ok := caCertPool.AppendCertsFromPEM(caCertContents); !ok { + return nil, fmt.Errorf("Could not append CA certs from %s", config.CACertPath) + } } tlsConfig = &tls.Config{ - Certificates: []tls.Certificate{cert}, + Certificates: certs, RootCAs: caCertPool, InsecureSkipVerify: config.SkipVerify, ServerName: config.ServerName, diff --git a/pkg/config/cluster.go b/pkg/config/cluster.go index 3c56df21..b73d6a4f 100644 --- a/pkg/config/cluster.go +++ b/pkg/config/cluster.go @@ -86,7 +86,7 @@ type ClusterSpec struct { } type ClientAuth struct { - UseTLS bool `json:"useTLS"` + TLSEnabled bool `json:"tlsEnabled"` CACertPath string `json:"caCertPath"` CertPath string `json:"certPath"` KeyPath string `json:"keyPath"` @@ -127,7 +127,7 @@ func (c ClusterConfig) Validate() error { ) } - if c.Spec.ClientAuth.UseTLS && len(c.Spec.ZKAddrs) > 0 { + if c.Spec.ClientAuth.TLSEnabled && len(c.Spec.ZKAddrs) > 0 { err = multierror.Append( err, errors.New("TLS not supported with zk access mode; omit zk addresses to fix"), @@ -158,7 +158,7 @@ func (c ClusterConfig) NewAdminClient( admin.BrokerAdminClientConfig{ ConnectorConfig: admin.ConnectorConfig{ BrokerAddr: c.Spec.BootstrapAddrs[0], - UseTLS: c.Spec.ClientAuth.UseTLS, + TLSEnabled: c.Spec.ClientAuth.TLSEnabled, CACertPath: c.absPath(c.Spec.ClientAuth.CACertPath), CertPath: c.absPath(c.Spec.ClientAuth.CertPath), KeyPath: c.absPath(c.Spec.ClientAuth.KeyPath), From c5620822fbd9f30bb8c30449f39f6a536cd39210 Mon Sep 17 00:00:00 2001 From: Benjamin Yolken Date: Tue, 24 Nov 2020 22:01:40 -0800 Subject: [PATCH 09/19] Update README --- README.md | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index e70a7af6..e0b165d5 100644 --- a/README.md +++ b/README.md @@ -402,19 +402,17 @@ operations, >= 2.4 for applies). ### TLS -TLS is supported when running `topicctl` in the exclusive broker API mode. To use this, you'll -need the following, each in PEM format: +TLS is supported when running `topicctl` in the exclusive broker API mode. To use this, either +set `--tls-enabled` in the command-line or, if using a cluster config, set `tlsEnabled: true` +in the `clientAuth` section of the latter. -1. Client certificate -2. Client private key -3. CA certificate(s) +In addition to standard TLS, the tool also supports mutual TLS using custom certs, keys, and CA +certs (in PEM format). As with the enabling of TLS, these can be configured either on the +command-line or in a cluster config. See [this config](examples/tls/cluster.yaml) for an example. -These can be extracted from Java keystores (like the ones used by the brokers) using the `keytool` -command. +### SASL -You can then specify the paths to these via either the `--tls-*` args on the command line or -via the `clientAuth` section in a cluster config spec. See [this config](examples/tls/cluster.yaml) -for an example of the latter. +Coming soon. ## Development From 29c7825519363d6deb4d48e95e7b10f225fdcedf Mon Sep 17 00:00:00 2001 From: Benjamin Yolken Date: Tue, 24 Nov 2020 22:10:16 -0800 Subject: [PATCH 10/19] Update default kafka version to 2.4.1 --- README.md | 13 +++++++------ docker-compose.yml | 18 ++++++++++-------- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index e0b165d5..e4289cef 100644 --- a/README.md +++ b/README.md @@ -200,12 +200,12 @@ only. ### Specifying the target cluster -There are two patterns for specifying a target cluster in the `topicctl` subcommands: +There are three ways to specify a target cluster in the `topicctl` subcommands: 1. `--cluster-config=[path]`, where the refererenced path is a cluster configuration - in the format expected by the `apply` command described above *or* -2. `--zk-addr=[zookeeper address]` and `--zk-prefix=[optional prefix for cluster in zookeeper]` -3. `--broker-addr=[broker address]` + in the format expected by the `apply` command described above, +2. `--zk-addr=[zookeeper address]` and `--zk-prefix=[optional prefix for cluster in zookeeper]`, *or* +3. `--broker-addr=[bootstrap broker address]` All subcommands support the `cluster-config` pattern. The last two are also supported by the `get`, `repl`, `reset-offsets`, and `tail` subcommands since these can be run @@ -241,8 +241,9 @@ spec: versionMajor: v0.10 # Version bootstrapAddrs: # One or more broker bootstrap addresses - my-cluster.example.com:9092 - zkAddrs: # One or more cluster zookeeper addresses - - zk.example.com:2181 + zkAddrs: # One or more cluster zookeeper addresses; if these are + - zk.example.com:2181 # omitted, then the cluster will only be accessed via broker APIs; + # see the section below on cluster access for more details. zkPrefix: my-cluster # Prefix for zookeeper nodes zkLockPath: /topicctl/locks # Path used for apply locks (optional) clusterID: abc-123-xyz # Expected cluster ID for cluster (optional, used as diff --git a/docker-compose.yml b/docker-compose.yml index 0db41fc2..99a5e5c6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,9 +1,11 @@ -# By default, this docker-compose setup uses Kafka 0.10.2. This version can +# By default, this docker-compose setup uses Kafka 2.4.1. This version can # be overwritten by setting the KAFKA_IMAGE_TAG environment variable; some choices here include: # # 1. Kafka 2.6: 2.13-2.6.0 # 2. Kafka 2.5: 2.13-2.5.0 -# 3. Kafka 2.4: 2.12-2.4.1 +# 3. Kafka 0.10: 2.11-0.10.2.2 +# +# See https://hub.docker.com/r/wurstmeister/kafka/tags for the complete list. version: '2.1' services: zookeeper: @@ -13,7 +15,7 @@ services: # Zone 1 brokers kafka1: - image: wurstmeister/kafka:${KAFKA_IMAGE_TAG:-2.11-0.10.2.2} + image: wurstmeister/kafka:${KAFKA_IMAGE_TAG:-2.12-2.4.1} ports: - "9092:9092" environment: @@ -27,7 +29,7 @@ services: - zookeeper kafka2: - image: wurstmeister/kafka:${KAFKA_IMAGE_TAG:-2.11-0.10.2.2} + image: wurstmeister/kafka:${KAFKA_IMAGE_TAG:-2.12-2.4.1} ports: - "9093:9092" environment: @@ -43,7 +45,7 @@ services: # Zone 2 brokers kafka3: - image: wurstmeister/kafka:${KAFKA_IMAGE_TAG:-2.11-0.10.2.2} + image: wurstmeister/kafka:${KAFKA_IMAGE_TAG:-2.12-2.4.1} ports: - "9094:9092" environment: @@ -59,7 +61,7 @@ services: - kafka2 kafka4: - image: wurstmeister/kafka:${KAFKA_IMAGE_TAG:-2.11-0.10.2.2} + image: wurstmeister/kafka:${KAFKA_IMAGE_TAG:-2.12-2.4.1} ports: - "9095:9092" environment: @@ -77,7 +79,7 @@ services: # Zone 3 brokers kafka5: - image: wurstmeister/kafka:${KAFKA_IMAGE_TAG:-2.11-0.10.2.2} + image: wurstmeister/kafka:${KAFKA_IMAGE_TAG:-2.12-2.4.1} ports: - "9096:9092" environment: @@ -95,7 +97,7 @@ services: - kafka4 kafka6: - image: wurstmeister/kafka:${KAFKA_IMAGE_TAG:-2.11-0.10.2.2} + image: wurstmeister/kafka:${KAFKA_IMAGE_TAG:-2.12-2.4.1} ports: - "9097:9092" environment: From 92fabba3de539018e6d8f926c402633f6b464b2d Mon Sep 17 00:00:00 2001 From: Benjamin Yolken Date: Wed, 25 Nov 2020 13:32:44 -0800 Subject: [PATCH 11/19] Update kafka-go version and fix tests --- docker-compose-tls.yml | 2 +- examples/{tls => auth}/certs/ca.crt | 0 examples/{tls => auth}/certs/ca.key | 0 examples/{tls => auth}/certs/client.crt | 0 examples/{tls => auth}/certs/client.key | 0 .../{tls => auth}/certs/kafka.keystore.jks | Bin .../{tls => auth}/certs/kafka.truststore.jks | Bin examples/{tls => auth}/cluster.yaml | 4 +-- .../{tls => auth}/topics/topic-default.yaml | 0 go.mod | 2 +- go.sum | 2 ++ pkg/admin/brokerclient.go | 28 ++++-------------- pkg/config/cluster.go | 22 +++++++------- pkg/config/cluster_test.go | 2 +- pkg/config/load_test.go | 4 +++ 15 files changed, 28 insertions(+), 38 deletions(-) rename examples/{tls => auth}/certs/ca.crt (100%) rename examples/{tls => auth}/certs/ca.key (100%) rename examples/{tls => auth}/certs/client.crt (100%) rename examples/{tls => auth}/certs/client.key (100%) rename examples/{tls => auth}/certs/kafka.keystore.jks (100%) rename examples/{tls => auth}/certs/kafka.truststore.jks (100%) rename examples/{tls => auth}/cluster.yaml (90%) rename examples/{tls => auth}/topics/topic-default.yaml (100%) diff --git a/docker-compose-tls.yml b/docker-compose-tls.yml index beeb85f4..f7bb8596 100644 --- a/docker-compose-tls.yml +++ b/docker-compose-tls.yml @@ -36,4 +36,4 @@ services: KAFKA_SECURITY_INTER_BROKER_PROTOCOL: 'SSL' volumes: - /var/run/docker.sock:/var/run/docker.sock - - ./examples/tls/certs:/certs + - ./examples/auth/certs:/certs diff --git a/examples/tls/certs/ca.crt b/examples/auth/certs/ca.crt similarity index 100% rename from examples/tls/certs/ca.crt rename to examples/auth/certs/ca.crt diff --git a/examples/tls/certs/ca.key b/examples/auth/certs/ca.key similarity index 100% rename from examples/tls/certs/ca.key rename to examples/auth/certs/ca.key diff --git a/examples/tls/certs/client.crt b/examples/auth/certs/client.crt similarity index 100% rename from examples/tls/certs/client.crt rename to examples/auth/certs/client.crt diff --git a/examples/tls/certs/client.key b/examples/auth/certs/client.key similarity index 100% rename from examples/tls/certs/client.key rename to examples/auth/certs/client.key diff --git a/examples/tls/certs/kafka.keystore.jks b/examples/auth/certs/kafka.keystore.jks similarity index 100% rename from examples/tls/certs/kafka.keystore.jks rename to examples/auth/certs/kafka.keystore.jks diff --git a/examples/tls/certs/kafka.truststore.jks b/examples/auth/certs/kafka.truststore.jks similarity index 100% rename from examples/tls/certs/kafka.truststore.jks rename to examples/auth/certs/kafka.truststore.jks diff --git a/examples/tls/cluster.yaml b/examples/auth/cluster.yaml similarity index 90% rename from examples/tls/cluster.yaml rename to examples/auth/cluster.yaml index 71864af0..b38bbb80 100644 --- a/examples/tls/cluster.yaml +++ b/examples/auth/cluster.yaml @@ -9,8 +9,8 @@ spec: versionMajor: v2 bootstrapAddrs: - localhost:9093 - clientAuth: - useTLS: true + tls: + enabled: true caCertPath: certs/ca.crt certPath: certs/client.crt keyPath: certs/client.key diff --git a/examples/tls/topics/topic-default.yaml b/examples/auth/topics/topic-default.yaml similarity index 100% rename from examples/tls/topics/topic-default.yaml rename to examples/auth/topics/topic-default.yaml diff --git a/go.mod b/go.mod index 43a93604..786a24ac 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( // This is a draft kafka-go version that is not merged into master of that // repo yet. - github.com/segmentio/kafka-go v0.4.9-0.20201120053500-98b10c7c5631 + github.com/segmentio/kafka-go v0.4.9-0.20201125211318-63239f1766cc github.com/sirupsen/logrus v1.2.0 github.com/spf13/cobra v1.0.0 diff --git a/go.sum b/go.sum index 77435739..7cfc5394 100644 --- a/go.sum +++ b/go.sum @@ -153,6 +153,8 @@ github.com/segmentio/kafka-go v0.4.9-0.20201119185034-ed175e9082b6 h1:kkk2CI7oly github.com/segmentio/kafka-go v0.4.9-0.20201119185034-ed175e9082b6/go.mod h1:Inh7PqOsxmfgasV8InZYKVXWsdjcCq2d9tFV75GLbuM= github.com/segmentio/kafka-go v0.4.9-0.20201120053500-98b10c7c5631 h1:BaADAfG3xFHYQ9CZ4kZITBSosvg9Wz0PLhxQFFDQvTI= github.com/segmentio/kafka-go v0.4.9-0.20201120053500-98b10c7c5631/go.mod h1:Inh7PqOsxmfgasV8InZYKVXWsdjcCq2d9tFV75GLbuM= +github.com/segmentio/kafka-go v0.4.9-0.20201125211318-63239f1766cc h1:hwPom0mtHZTB+96qMCt8QxjhDHZZ15+4eGuM0DdCKUY= +github.com/segmentio/kafka-go v0.4.9-0.20201125211318-63239f1766cc/go.mod h1:Inh7PqOsxmfgasV8InZYKVXWsdjcCq2d9tFV75GLbuM= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= diff --git a/pkg/admin/brokerclient.go b/pkg/admin/brokerclient.go index aad8a94b..60f767ae 100644 --- a/pkg/admin/brokerclient.go +++ b/pkg/admin/brokerclient.go @@ -47,7 +47,7 @@ func NewBrokerAdminClient( return nil, err } log.Debugf("Supported API versions: %+v", apiVersions) - maxVersions := map[string]int16{} + maxVersions := map[string]int{} for _, apiKey := range apiVersions.ApiKeys { maxVersions[apiKey.ApiName] = apiKey.MaxVersion } @@ -370,15 +370,9 @@ func (c *BrokerAdminClient) AssignPartitions( apiAssignments := []kafka.AlterPartitionReassignmentsRequestAssignment{} for _, assignment := range assignments { - replicas := []int32{} - - for _, replica := range assignment.Replicas { - replicas = append(replicas, int32(replica)) - } - apiAssignment := kafka.AlterPartitionReassignmentsRequestAssignment{ - PartitionID: int32(assignment.ID), - BrokerIDs: replicas, + PartitionID: assignment.ID, + BrokerIDs: assignment.Replicas, } apiAssignments = append(apiAssignments, apiAssignment) } @@ -411,15 +405,10 @@ func (c *BrokerAdminClient) AddPartitions( partitions := []kafka.CreatePartitionsRequestPartition{} for _, newAssignment := range newAssignments { - replicas := []int32{} - for _, replica := range newAssignment.Replicas { - replicas = append(replicas, int32(replica)) - } - partitions = append( partitions, kafka.CreatePartitionsRequestPartition{ - BrokerIDs: replicas, + BrokerIDs: newAssignment.Replicas, }, ) } @@ -427,7 +416,7 @@ func (c *BrokerAdminClient) AddPartitions( req := kafka.CreatePartitionsRequest{ Topic: topic, NewPartitions: partitions, - TotalCount: int32(len(partitions) + len(topicInfo.Partitions)), + TotalCount: len(partitions) + len(topicInfo.Partitions), Timeout: defaultTimeout, } log.Debugf("CreatePartitions request: %+v", req) @@ -447,14 +436,9 @@ func (c *BrokerAdminClient) RunLeaderElection( return errors.New("Cannot run leader election in read-only mode") } - partitionsInt32 := []int32{} - for _, partition := range partitions { - partitionsInt32 = append(partitionsInt32, int32(partition)) - } - req := kafka.ElectLeadersRequest{ Topic: topic, - Partitions: partitionsInt32, + Partitions: partitions, Timeout: defaultTimeout, } log.Debugf("ElectLeaders request: %+v", req) diff --git a/pkg/config/cluster.go b/pkg/config/cluster.go index b73d6a4f..ce2af374 100644 --- a/pkg/config/cluster.go +++ b/pkg/config/cluster.go @@ -80,13 +80,13 @@ type ClusterSpec struct { // limited by. If unset, no retention drop limiting will be applied. DefaultRetentionDropStepDurationStr string `json:"defaultRetentionDropStepDuration"` - // ClientAuth stores how we should authenticate broker connections, if appropriate. Only + // TLS stores how we should use TLS with broker connections, if appropriate. Only // applies if using the broker admin. - ClientAuth ClientAuth `json:"clientAuth"` + TLS TLSConfig `json:"tls"` } -type ClientAuth struct { - TLSEnabled bool `json:"tlsEnabled"` +type TLSConfig struct { + Enabled bool `json:"enabled"` CACertPath string `json:"caCertPath"` CertPath string `json:"certPath"` KeyPath string `json:"keyPath"` @@ -127,7 +127,7 @@ func (c ClusterConfig) Validate() error { ) } - if c.Spec.ClientAuth.TLSEnabled && len(c.Spec.ZKAddrs) > 0 { + if c.Spec.TLS.Enabled && len(c.Spec.ZKAddrs) > 0 { err = multierror.Append( err, errors.New("TLS not supported with zk access mode; omit zk addresses to fix"), @@ -158,12 +158,12 @@ func (c ClusterConfig) NewAdminClient( admin.BrokerAdminClientConfig{ ConnectorConfig: admin.ConnectorConfig{ BrokerAddr: c.Spec.BootstrapAddrs[0], - TLSEnabled: c.Spec.ClientAuth.TLSEnabled, - CACertPath: c.absPath(c.Spec.ClientAuth.CACertPath), - CertPath: c.absPath(c.Spec.ClientAuth.CertPath), - KeyPath: c.absPath(c.Spec.ClientAuth.KeyPath), - ServerName: c.Spec.ClientAuth.ServerName, - SkipVerify: c.Spec.ClientAuth.SkipVerify, + TLSEnabled: c.Spec.TLS.Enabled, + CACertPath: c.absPath(c.Spec.TLS.CACertPath), + CertPath: c.absPath(c.Spec.TLS.CertPath), + KeyPath: c.absPath(c.Spec.TLS.KeyPath), + ServerName: c.Spec.TLS.ServerName, + SkipVerify: c.Spec.TLS.SkipVerify, }, ReadOnly: readOnly, }, diff --git a/pkg/config/cluster_test.go b/pkg/config/cluster_test.go index 9e0b3a40..db5bfee7 100644 --- a/pkg/config/cluster_test.go +++ b/pkg/config/cluster_test.go @@ -77,7 +77,7 @@ func TestClusterValidate(t *testing.T) { VersionMajor: "v2", }, }, - expError: true, + expError: false, }, { description: "bad retention drop format", diff --git a/pkg/config/load_test.go b/pkg/config/load_test.go index bd1bb4c7..16ed7de7 100644 --- a/pkg/config/load_test.go +++ b/pkg/config/load_test.go @@ -10,6 +10,10 @@ import ( func TestLoadCluster(t *testing.T) { clusterConfig, err := LoadClusterFile("testdata/test-cluster/cluster.yaml") assert.Nil(t, err) + + // Empty RootDir since this will vary based on where test is run. + clusterConfig.RootDir = "" + assert.Equal( t, ClusterConfig{ From ac10325f263bf304a40e2477bae6485e349aeea4 Mon Sep 17 00:00:00 2001 From: Benjamin Yolken Date: Wed, 25 Nov 2020 16:35:35 -0800 Subject: [PATCH 12/19] Clean up SASL implementation --- cmd/topicctl/subcmd/apply.go | 1 + cmd/topicctl/subcmd/get.go | 137 ++-------------------- cmd/topicctl/subcmd/repl.go | 136 +--------------------- cmd/topicctl/subcmd/reset.go | 137 ++-------------------- cmd/topicctl/subcmd/root.go | 33 ------ cmd/topicctl/subcmd/shared.go | 211 ++++++++++++++++++++++++++++++++++ cmd/topicctl/subcmd/tail.go | 141 ++--------------------- cmd/topicctl/subcmd/tester.go | 115 +++--------------- docker-compose-sasl.yml | 44 +++++++ docker-compose-tls.yml | 20 ++-- go.sum | 1 + pkg/admin/brokerclient.go | 1 + pkg/admin/connector.go | 100 +++++++++++++--- pkg/config/cluster.go | 42 ++++++- 14 files changed, 437 insertions(+), 682 deletions(-) create mode 100644 cmd/topicctl/subcmd/shared.go create mode 100644 docker-compose-sasl.yml diff --git a/cmd/topicctl/subcmd/apply.go b/cmd/topicctl/subcmd/apply.go index b04fb2e0..96139b1c 100644 --- a/cmd/topicctl/subcmd/apply.go +++ b/cmd/topicctl/subcmd/apply.go @@ -104,6 +104,7 @@ func init() { "Amount of time to wait between partition checks", ) + applyCmd.MarkFlagRequired("cluster-config") RootCmd.AddCommand(applyCmd) } diff --git a/cmd/topicctl/subcmd/get.go b/cmd/topicctl/subcmd/get.go index f54072a1..69c2ffd9 100644 --- a/cmd/topicctl/subcmd/get.go +++ b/cmd/topicctl/subcmd/get.go @@ -3,13 +3,10 @@ package subcmd import ( "context" "fmt" - "os" "strings" "github.com/aws/aws-sdk-go/aws/session" - "github.com/segmentio/topicctl/pkg/admin" "github.com/segmentio/topicctl/pkg/cli" - "github.com/segmentio/topicctl/pkg/config" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" ) @@ -32,155 +29,35 @@ var getCmd = &cobra.Command{ } type getCmdConfig struct { - brokerAddr string - clusterConfig string - full bool - tlsCACert string - tlsCert string - tlsEnabled bool - tlsKey string - tlsSkipVerify bool - tlsServerName string - zkAddr string - zkPrefix string + full bool + shared sharedOptions } var getConfig getCmdConfig func init() { - getCmd.Flags().StringVarP( - &getConfig.brokerAddr, - "broker-addr", - "b", - "", - "Broker address", - ) - getCmd.Flags().StringVar( - &getConfig.clusterConfig, - "cluster-config", - os.Getenv("TOPICCTL_CLUSTER_CONFIG"), - "Cluster config", - ) getCmd.Flags().BoolVar( &getConfig.full, "full", false, "Show more full information for resources", ) - getCmd.Flags().StringVar( - &getConfig.tlsCACert, - "tls-ca-cert", - "", - "Path to client CA cert PEM file if using TLS", - ) - getCmd.Flags().StringVar( - &getConfig.tlsCert, - "tls-cert", - "", - "Path to client cert PEM file if using TLS", - ) - getCmd.Flags().BoolVar( - &getConfig.tlsEnabled, - "tls-enabled", - false, - "Use TLS for communication with brokers", - ) - getCmd.Flags().StringVar( - &getConfig.tlsKey, - "tls-key", - "", - "Path to client private key PEM file if using TLS", - ) - getCmd.Flags().StringVar( - &getConfig.tlsServerName, - "tls-server-name", - "", - "Server name to use for TLS cert verification", - ) - getCmd.Flags().BoolVar( - &getConfig.tlsSkipVerify, - "tls-skip-verify", - false, - "Skip hostname verification when using TLS", - ) - getCmd.Flags().StringVarP( - &getConfig.zkAddr, - "zk-addr", - "z", - "", - "ZooKeeper address", - ) - getCmd.Flags().StringVar( - &getConfig.zkPrefix, - "zk-prefix", - "", - "Prefix for cluster-related nodes in zk", - ) + addSharedFlags(getCmd, &getConfig.shared) RootCmd.AddCommand(getCmd) } func getPreRun(cmd *cobra.Command, args []string) error { - return validateCommonFlags( - getConfig.clusterConfig, - getConfig.zkAddr, - getConfig.zkPrefix, - getConfig.brokerAddr, - getConfig.tlsCACert, - getConfig.tlsCert, - getConfig.tlsKey, - ) + return getConfig.shared.validate() } func getRun(cmd *cobra.Command, args []string) error { ctx := context.Background() sess := session.Must(session.NewSession()) - var adminClient admin.Client - var clientErr error - - if getConfig.clusterConfig != "" { - clusterConfig, err := config.LoadClusterFile(getConfig.clusterConfig) - if err != nil { - return err - } - adminClient, clientErr = clusterConfig.NewAdminClient(ctx, sess, true) - } else if getConfig.brokerAddr != "" { - tlsEnabled := (getConfig.tlsEnabled || - getConfig.tlsCACert != "" || - getConfig.tlsCert != "" || - getConfig.tlsKey != "") - adminClient, clientErr = admin.NewBrokerAdminClient( - ctx, - admin.BrokerAdminClientConfig{ - ConnectorConfig: admin.ConnectorConfig{ - BrokerAddr: getConfig.brokerAddr, - TLSEnabled: tlsEnabled, - CACertPath: getConfig.tlsCACert, - CertPath: getConfig.tlsCert, - KeyPath: getConfig.tlsKey, - ServerName: getConfig.tlsServerName, - SkipVerify: getConfig.tlsSkipVerify, - }, - ReadOnly: true, - }, - ) - } else { - adminClient, clientErr = admin.NewZKAdminClient( - ctx, - admin.ZKAdminClientConfig{ - ZKAddrs: []string{getConfig.zkAddr}, - ZKPrefix: getConfig.zkPrefix, - Sess: sess, - // Run in read-only mode to ensure that tailing doesn't make any changes - // in the cluster - ReadOnly: true, - }, - ) - } - - if clientErr != nil { - return clientErr + adminClient, err := getConfig.shared.getAdminClient(ctx, sess, true) + if err != nil { + return err } defer adminClient.Close() diff --git a/cmd/topicctl/subcmd/repl.go b/cmd/topicctl/subcmd/repl.go index 2ed03e80..c3f8aaf5 100644 --- a/cmd/topicctl/subcmd/repl.go +++ b/cmd/topicctl/subcmd/repl.go @@ -2,12 +2,9 @@ package subcmd import ( "context" - "os" "github.com/aws/aws-sdk-go/aws/session" - "github.com/segmentio/topicctl/pkg/admin" "github.com/segmentio/topicctl/pkg/cli" - "github.com/segmentio/topicctl/pkg/config" "github.com/spf13/cobra" ) @@ -19,148 +16,27 @@ var replCmd = &cobra.Command{ } type replCmdConfig struct { - brokerAddr string - clusterConfig string - tlsCACert string - tlsCert string - tlsEnabled bool - tlsKey string - tlsSkipVerify bool - tlsServerName string - zkAddr string - zkPrefix string + shared sharedOptions } var replConfig replCmdConfig func init() { - replCmd.Flags().StringVarP( - &replConfig.brokerAddr, - "broker-addr", - "b", - "", - "Broker address", - ) - replCmd.Flags().StringVar( - &replConfig.clusterConfig, - "cluster-config", - os.Getenv("TOPICCTL_CLUSTER_CONFIG"), - "Cluster config", - ) - replCmd.Flags().StringVar( - &replConfig.tlsCACert, - "tls-ca-cert", - "", - "Path to client CA cert PEM file if using TLS", - ) - replCmd.Flags().StringVar( - &replConfig.tlsCert, - "tls-cert", - "", - "Path to client cert PEM file if using TLS", - ) - replCmd.Flags().BoolVar( - &replConfig.tlsEnabled, - "tls-enabled", - false, - "Use TLS for communication with brokers", - ) - replCmd.Flags().StringVar( - &replConfig.tlsKey, - "tls-key", - "", - "Path to client private key PEM file if using TLS", - ) - replCmd.Flags().StringVar( - &replConfig.tlsServerName, - "tls-server-name", - "", - "Server name to use for TLS cert verification", - ) - replCmd.Flags().BoolVar( - &replConfig.tlsSkipVerify, - "tls-skip-verify", - false, - "Skip hostname verification when using TLS", - ) - replCmd.Flags().StringVarP( - &replConfig.zkAddr, - "zk-addr", - "z", - "", - "ZooKeeper address", - ) - replCmd.Flags().StringVar( - &replConfig.zkPrefix, - "zk-prefix", - "", - "Prefix for cluster-related nodes in zk", - ) - + addSharedFlags(replCmd, &replConfig.shared) RootCmd.AddCommand(replCmd) } func replPreRun(cmd *cobra.Command, args []string) error { - return validateCommonFlags( - replConfig.clusterConfig, - replConfig.zkAddr, - replConfig.zkPrefix, - replConfig.brokerAddr, - replConfig.tlsCACert, - replConfig.tlsCert, - replConfig.tlsKey, - ) + return replConfig.shared.validate() } func replRun(cmd *cobra.Command, args []string) error { ctx := context.Background() sess := session.Must(session.NewSession()) - var adminClient admin.Client - var clientErr error - - if replConfig.clusterConfig != "" { - clusterConfig, err := config.LoadClusterFile(replConfig.clusterConfig) - if err != nil { - return err - } - adminClient, clientErr = clusterConfig.NewAdminClient(ctx, sess, true) - } else if replConfig.brokerAddr != "" { - tlsEnabled := (replConfig.tlsEnabled || - replConfig.tlsCACert != "" || - replConfig.tlsCert != "" || - replConfig.tlsKey != "") - adminClient, clientErr = admin.NewBrokerAdminClient( - ctx, - admin.BrokerAdminClientConfig{ - ConnectorConfig: admin.ConnectorConfig{ - BrokerAddr: replConfig.brokerAddr, - TLSEnabled: tlsEnabled, - CACertPath: replConfig.tlsCACert, - CertPath: replConfig.tlsCert, - KeyPath: replConfig.tlsKey, - ServerName: replConfig.tlsServerName, - SkipVerify: replConfig.tlsSkipVerify, - }, - ReadOnly: true, - }, - ) - } else { - adminClient, clientErr = admin.NewZKAdminClient( - ctx, - admin.ZKAdminClientConfig{ - ZKAddrs: []string{replConfig.zkAddr}, - ZKPrefix: replConfig.zkPrefix, - Sess: sess, - // Run in read-only mode to ensure that tailing doesn't make any changes - // in the cluster - ReadOnly: true, - }, - ) - } - - if clientErr != nil { - return clientErr + adminClient, err := replConfig.shared.getAdminClient(ctx, sess, true) + if err != nil { + return err } defer adminClient.Close() diff --git a/cmd/topicctl/subcmd/reset.go b/cmd/topicctl/subcmd/reset.go index b3a1892e..39cc9c87 100644 --- a/cmd/topicctl/subcmd/reset.go +++ b/cmd/topicctl/subcmd/reset.go @@ -4,12 +4,9 @@ import ( "context" "errors" "fmt" - "os" - "github.com/segmentio/topicctl/pkg/admin" "github.com/segmentio/topicctl/pkg/apply" "github.com/segmentio/topicctl/pkg/cli" - "github.com/segmentio/topicctl/pkg/config" "github.com/segmentio/topicctl/pkg/groups" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" @@ -24,36 +21,15 @@ var resetOffsetsCmd = &cobra.Command{ } type resetOffsetsCmdConfig struct { - brokerAddr string - clusterConfig string - offset int64 - partitions []int - tlsCACert string - tlsCert string - tlsEnabled bool - tlsKey string - tlsSkipVerify bool - tlsServerName string - zkAddr string - zkPrefix string + offset int64 + partitions []int + + shared sharedOptions } var resetOffsetsConfig resetOffsetsCmdConfig func init() { - resetOffsetsCmd.Flags().StringVarP( - &resetOffsetsConfig.brokerAddr, - "broker-addr", - "b", - "", - "Broker address", - ) - resetOffsetsCmd.Flags().StringVar( - &resetOffsetsConfig.clusterConfig, - "cluster-config", - os.Getenv("TOPICCTL_CLUSTER_CONFIG"), - "Cluster config", - ) resetOffsetsCmd.Flags().Int64Var( &resetOffsetsConfig.offset, "offset", @@ -66,117 +42,22 @@ func init() { []int{}, "Partition (defaults to all)", ) - resetOffsetsCmd.Flags().StringVar( - &resetOffsetsConfig.tlsCACert, - "tls-ca-cert", - "", - "Path to client CA cert PEM file if using TLS", - ) - resetOffsetsCmd.Flags().StringVar( - &resetOffsetsConfig.tlsCert, - "tls-cert", - "", - "Path to client cert PEM file if using TLS", - ) - resetOffsetsCmd.Flags().BoolVar( - &resetOffsetsConfig.tlsEnabled, - "tls-enabled", - false, - "Use TLS for communication with brokers", - ) - resetOffsetsCmd.Flags().StringVar( - &resetOffsetsConfig.tlsKey, - "tls-key", - "", - "Path to client private key PEM file if using TLS", - ) - resetOffsetsCmd.Flags().StringVar( - &resetOffsetsConfig.tlsServerName, - "tls-server-name", - "", - "Server name to use for TLS cert verification", - ) - resetOffsetsCmd.Flags().BoolVar( - &resetOffsetsConfig.tlsSkipVerify, - "tls-skip-verify", - false, - "Skip hostname verification when using TLS", - ) - resetOffsetsCmd.Flags().StringVarP( - &resetOffsetsConfig.zkAddr, - "zk-addr", - "z", - "", - "ZooKeeper address", - ) - resetOffsetsCmd.Flags().StringVar( - &resetOffsetsConfig.zkPrefix, - "zk-prefix", - "", - "Prefix for cluster-related nodes in zk", - ) + addSharedFlags(resetOffsetsCmd, &resetOffsetsConfig.shared) RootCmd.AddCommand(resetOffsetsCmd) } func resetOffsetsPreRun(cmd *cobra.Command, args []string) error { - return validateCommonFlags( - resetOffsetsConfig.clusterConfig, - resetOffsetsConfig.zkAddr, - resetOffsetsConfig.zkPrefix, - resetOffsetsConfig.brokerAddr, - resetOffsetsConfig.tlsCACert, - resetOffsetsConfig.tlsCert, - resetOffsetsConfig.tlsKey, - ) + return resetOffsetsConfig.shared.validate() } func resetOffsetsRun(cmd *cobra.Command, args []string) error { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - var adminClient admin.Client - var clientErr error - - if resetOffsetsConfig.clusterConfig != "" { - clusterConfig, err := config.LoadClusterFile(resetOffsetsConfig.clusterConfig) - if err != nil { - return err - } - adminClient, clientErr = clusterConfig.NewAdminClient(ctx, nil, false) - } else if resetOffsetsConfig.brokerAddr != "" { - tlsEnabled := (resetOffsetsConfig.tlsEnabled || - resetOffsetsConfig.tlsCACert != "" || - resetOffsetsConfig.tlsCert != "" || - resetOffsetsConfig.tlsKey != "") - adminClient, clientErr = admin.NewBrokerAdminClient( - ctx, - admin.BrokerAdminClientConfig{ - ConnectorConfig: admin.ConnectorConfig{ - BrokerAddr: resetOffsetsConfig.brokerAddr, - TLSEnabled: tlsEnabled, - CACertPath: resetOffsetsConfig.tlsCACert, - CertPath: resetOffsetsConfig.tlsCert, - KeyPath: resetOffsetsConfig.tlsKey, - ServerName: resetOffsetsConfig.tlsServerName, - SkipVerify: resetOffsetsConfig.tlsSkipVerify, - }, - ReadOnly: true, - }, - ) - } else { - adminClient, clientErr = admin.NewZKAdminClient( - ctx, - admin.ZKAdminClientConfig{ - ZKAddrs: []string{resetOffsetsConfig.zkAddr}, - ZKPrefix: resetOffsetsConfig.zkPrefix, - ReadOnly: false, - }, - ) - } - - if clientErr != nil { - return clientErr + adminClient, err := replConfig.shared.getAdminClient(ctx, nil, true) + if err != nil { + return err } defer adminClient.Close() diff --git a/cmd/topicctl/subcmd/root.go b/cmd/topicctl/subcmd/root.go index 6292632f..ad691b54 100644 --- a/cmd/topicctl/subcmd/root.go +++ b/cmd/topicctl/subcmd/root.go @@ -1,7 +1,6 @@ package subcmd import ( - "errors" "fmt" "os" @@ -59,35 +58,3 @@ func preRun(cmd *cobra.Command, args []string) error { } return nil } - -func validateCommonFlags( - clusterConfig string, - zkAddr string, - zkPrefix string, - brokerAddr string, - tlsCACert string, - tlsCert string, - tlsKey string, -) error { - if clusterConfig == "" && zkAddr == "" && brokerAddr == "" { - return errors.New("Must set either broker-addr, cluster-config, or zk-addr") - } - if zkAddr != "" && brokerAddr != "" { - return errors.New("Cannot set both zk-addr and broker-addr") - } - if clusterConfig != "" && - (zkAddr != "" || zkPrefix != "" || brokerAddr != "" || tlsCACert != "" || - tlsCert != "" || tlsKey != "") { - log.Warn("Broker and zk flags are ignored when using cluster-config") - } - - useTLS := tlsCACert != "" || tlsCert != "" || tlsKey != "" - if useTLS && (tlsCACert == "" || tlsCert == "" || tlsKey == "") { - return errors.New("Must set tls-ca-cert, tls-cert, and tls-key if using TLS") - } - if useTLS && zkAddr != "" { - log.Warn("Auth flags are ignored accessing cluster via zookeeper") - } - - return nil -} diff --git a/cmd/topicctl/subcmd/shared.go b/cmd/topicctl/subcmd/shared.go new file mode 100644 index 00000000..6c8f2ee6 --- /dev/null +++ b/cmd/topicctl/subcmd/shared.go @@ -0,0 +1,211 @@ +package subcmd + +import ( + "context" + "errors" + "os" + + "github.com/aws/aws-sdk-go/aws/session" + "github.com/hashicorp/go-multierror" + "github.com/segmentio/topicctl/pkg/admin" + "github.com/segmentio/topicctl/pkg/config" + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" +) + +type sharedOptions struct { + brokerAddr string + clusterConfig string + saslMechanism string + saslPassword string + saslUsername string + tlsCACert string + tlsCert string + tlsEnabled bool + tlsKey string + tlsSkipVerify bool + tlsServerName string + zkAddr string + zkPrefix string +} + +func (s sharedOptions) validate() error { + var err error + + if s.clusterConfig == "" && s.zkAddr == "" && s.brokerAddr == "" { + err = multierror.Append( + err, + errors.New("Must set either broker-addr, cluster-config, or zk-addr"), + ) + } + + if s.zkAddr != "" && s.brokerAddr != "" { + err = multierror.Append( + err, + errors.New("Cannot set both zk-addr and broker-addr"), + ) + } + if s.clusterConfig != "" && + (s.zkAddr != "" || s.zkPrefix != "" || s.brokerAddr != "" || s.tlsCACert != "" || + s.tlsCert != "" || s.tlsKey != "" || s.saslMechanism != "") { + log.Warn("Broker and zk flags are ignored when using cluster-config") + } + + useTLS := s.tlsEnabled || s.tlsCACert != "" || s.tlsCert != "" || s.tlsKey != "" + useSASL := s.saslMechanism != "" || s.saslPassword != "" || s.saslUsername != "" + + if useTLS && s.zkAddr != "" { + log.Warn("TLS flags are ignored accessing cluster via zookeeper") + } + if useSASL && s.zkAddr != "" { + log.Warn("SASL flags are ignored accessing cluster via zookeeper") + } + + if useSASL { + if saslErr := admin.ValidateSASLMechanism(s.saslMechanism); saslErr != nil { + err = multierror.Append(err, saslErr) + } + } + + return err +} + +func (s sharedOptions) getAdminClient( + ctx context.Context, + sess *session.Session, + readOnly bool, +) (admin.Client, error) { + if s.clusterConfig != "" { + clusterConfig, err := config.LoadClusterFile(s.clusterConfig) + if err != nil { + return nil, err + } + return clusterConfig.NewAdminClient(ctx, sess, true) + } else if s.brokerAddr != "" { + tlsEnabled := (s.tlsEnabled || + s.tlsCACert != "" || + s.tlsCert != "" || + s.tlsKey != "") + saslEnabled := (s.saslMechanism != "" || + s.saslPassword != "" || + s.saslUsername != "") + return admin.NewBrokerAdminClient( + ctx, + admin.BrokerAdminClientConfig{ + ConnectorConfig: admin.ConnectorConfig{ + BrokerAddr: s.brokerAddr, + TLS: admin.TLSConfig{ + Enabled: tlsEnabled, + CACertPath: s.tlsCACert, + CertPath: s.tlsCert, + KeyPath: s.tlsKey, + ServerName: s.tlsServerName, + SkipVerify: s.tlsSkipVerify, + }, + SASL: admin.SASLConfig{ + Enabled: saslEnabled, + Mechanism: s.saslMechanism, + Password: s.saslPassword, + Username: s.saslUsername, + }, + }, + ReadOnly: readOnly, + }, + ) + } else { + return admin.NewZKAdminClient( + ctx, + admin.ZKAdminClientConfig{ + ZKAddrs: []string{s.zkAddr}, + ZKPrefix: s.zkPrefix, + Sess: sess, + // Run in read-only mode to ensure that tailing doesn't make any changes + // in the cluster + ReadOnly: readOnly, + }, + ) + } +} + +func addSharedFlags(cmd *cobra.Command, options *sharedOptions) { + cmd.Flags().StringVarP( + &options.brokerAddr, + "broker-addr", + "b", + "", + "Broker address", + ) + cmd.Flags().StringVar( + &options.clusterConfig, + "cluster-config", + os.Getenv("TOPICCTL_CLUSTER_CONFIG"), + "Cluster config", + ) + cmd.Flags().StringVar( + &options.saslMechanism, + "sasl-mechanism", + "", + "SASL mechanism if using SASL (either: plain, scram-sha-256, or scram-sha-512)", + ) + cmd.Flags().StringVar( + &options.saslPassword, + "sasl-password", + os.Getenv("TOPICCTL_SASL_PASSWORD"), + "SASL password if using SASL", + ) + cmd.Flags().StringVar( + &options.saslUsername, + "sasl-username", + os.Getenv("TOPICCTL_SASL_USERNAME"), + "SASL username if using SASL", + ) + cmd.Flags().StringVar( + &options.tlsCACert, + "tls-ca-cert", + "", + "Path to client CA cert PEM file if using TLS", + ) + cmd.Flags().StringVar( + &options.tlsCert, + "tls-cert", + "", + "Path to client cert PEM file if using TLS", + ) + cmd.Flags().BoolVar( + &options.tlsEnabled, + "tls-enabled", + false, + "Use TLS for communication with brokers", + ) + cmd.Flags().StringVar( + &options.tlsKey, + "tls-key", + "", + "Path to client private key PEM file if using TLS", + ) + cmd.Flags().StringVar( + &options.tlsServerName, + "tls-server-name", + "", + "Server name to use for TLS cert verification", + ) + cmd.Flags().BoolVar( + &options.tlsSkipVerify, + "tls-skip-verify", + false, + "Skip hostname verification when using TLS", + ) + cmd.Flags().StringVarP( + &options.zkAddr, + "zk-addr", + "z", + "", + "ZooKeeper address", + ) + cmd.Flags().StringVar( + &options.zkPrefix, + "zk-prefix", + "", + "Prefix for cluster-related nodes in zk", + ) +} diff --git a/cmd/topicctl/subcmd/tail.go b/cmd/topicctl/subcmd/tail.go index ecd19f5b..40a906c9 100644 --- a/cmd/topicctl/subcmd/tail.go +++ b/cmd/topicctl/subcmd/tail.go @@ -8,9 +8,7 @@ import ( "syscall" "github.com/segmentio/kafka-go" - "github.com/segmentio/topicctl/pkg/admin" "github.com/segmentio/topicctl/pkg/cli" - "github.com/segmentio/topicctl/pkg/config" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" ) @@ -24,37 +22,16 @@ var tailCmd = &cobra.Command{ } type tailCmdConfig struct { - brokerAddr string - clusterConfig string - offset int64 - partitions []int - raw bool - tlsCACert string - tlsCert string - tlsEnabled bool - tlsKey string - tlsSkipVerify bool - tlsServerName string - zkAddr string - zkPrefix string + offset int64 + partitions []int + raw bool + + shared sharedOptions } var tailConfig tailCmdConfig func init() { - tailCmd.Flags().StringVarP( - &tailConfig.brokerAddr, - "broker-addr", - "b", - "", - "Broker address", - ) - tailCmd.Flags().StringVar( - &tailConfig.clusterConfig, - "cluster-config", - os.Getenv("TOPICCTL_CLUSTER_CONFIG"), - "Cluster config", - ) tailCmd.Flags().Int64Var( &tailConfig.offset, "offset", @@ -73,56 +50,8 @@ func init() { false, "Output raw values only", ) - tailCmd.Flags().StringVar( - &tailConfig.tlsCACert, - "tls-ca-cert", - "", - "Path to client CA cert PEM file if using TLS", - ) - tailCmd.Flags().StringVar( - &tailConfig.tlsCert, - "tls-cert", - "", - "Path to client cert PEM file if using TLS", - ) - tailCmd.Flags().BoolVar( - &tailConfig.tlsEnabled, - "tls-enabled", - false, - "Use TLS for communication with brokers", - ) - tailCmd.Flags().StringVar( - &tailConfig.tlsKey, - "tls-key", - "", - "Path to client private key PEM file if using TLS", - ) - tailCmd.Flags().StringVar( - &tailConfig.tlsServerName, - "tls-server-name", - "", - "Server name to use for TLS cert verification", - ) - tailCmd.Flags().BoolVar( - &tailConfig.tlsSkipVerify, - "tls-skip-verify", - false, - "Skip hostname verification when using TLS", - ) - tailCmd.Flags().StringVarP( - &tailConfig.zkAddr, - "zk-addr", - "z", - "", - "ZooKeeper address", - ) - tailCmd.Flags().StringVar( - &tailConfig.zkPrefix, - "zk-prefix", - "", - "Prefix for cluster-related nodes in zk", - ) + addSharedFlags(tailCmd, &tailConfig.shared) RootCmd.AddCommand(tailCmd) } @@ -131,16 +60,7 @@ func tailPreRun(cmd *cobra.Command, args []string) error { // In raw mode, only log out errors log.SetLevel(log.ErrorLevel) } - - return validateCommonFlags( - tailConfig.clusterConfig, - tailConfig.zkAddr, - tailConfig.zkPrefix, - tailConfig.brokerAddr, - tailConfig.tlsCACert, - tailConfig.tlsCert, - tailConfig.tlsKey, - ) + return tailConfig.shared.validate() } func tailRun(cmd *cobra.Command, args []string) error { @@ -154,50 +74,9 @@ func tailRun(cmd *cobra.Command, args []string) error { cancel() }() - var adminClient admin.Client - var clientErr error - - if tailConfig.clusterConfig != "" { - clusterConfig, err := config.LoadClusterFile(tailConfig.clusterConfig) - if err != nil { - return err - } - adminClient, clientErr = clusterConfig.NewAdminClient(ctx, nil, true) - } else if tailConfig.brokerAddr != "" { - tlsEnabled := (tailConfig.tlsEnabled || - tailConfig.tlsCACert != "" || - tailConfig.tlsCert != "" || - tailConfig.tlsKey != "") - adminClient, clientErr = admin.NewBrokerAdminClient( - ctx, - admin.BrokerAdminClientConfig{ - ConnectorConfig: admin.ConnectorConfig{ - BrokerAddr: tailConfig.brokerAddr, - TLSEnabled: tlsEnabled, - CACertPath: tailConfig.tlsCACert, - CertPath: tailConfig.tlsCert, - KeyPath: tailConfig.tlsKey, - ServerName: tailConfig.tlsServerName, - SkipVerify: tailConfig.tlsSkipVerify, - }, - ReadOnly: true, - }, - ) - } else { - adminClient, clientErr = admin.NewZKAdminClient( - ctx, - admin.ZKAdminClientConfig{ - ZKAddrs: []string{tailConfig.zkAddr}, - ZKPrefix: tailConfig.zkPrefix, - // Run in read-only mode to ensure that tailing doesn't make any changes - // in the cluster - ReadOnly: true, - }, - ) - } - - if clientErr != nil { - return clientErr + adminClient, err := tailConfig.shared.getAdminClient(ctx, nil, true) + if err != nil { + return err } defer adminClient.Close() diff --git a/cmd/topicctl/subcmd/tester.go b/cmd/topicctl/subcmd/tester.go index 7c944d11..6e760a9f 100644 --- a/cmd/topicctl/subcmd/tester.go +++ b/cmd/topicctl/subcmd/tester.go @@ -10,7 +10,6 @@ import ( "time" "github.com/segmentio/kafka-go" - "github.com/segmentio/topicctl/pkg/admin" "github.com/segmentio/topicctl/pkg/apply" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" @@ -24,30 +23,17 @@ var testerCmd = &cobra.Command{ } type testerCmdConfig struct { - brokerAddr string - mode string - readConsumer string - tlsCACert string - tlsCert string - tlsEnabled bool - tlsKey string - tlsSkipVerify bool - tlsServerName string - topic string - writeRate int - zkAddr string + mode string + readConsumer string + topic string + writeRate int + + shared sharedOptions } var testerConfig testerCmdConfig func init() { - testerCmd.Flags().StringVarP( - &testerConfig.brokerAddr, - "broker-addr", - "b", - "", - "Broker address", - ) testerCmd.Flags().StringVar( &testerConfig.mode, "mode", @@ -60,42 +46,6 @@ func init() { "test-consumer", "Consumer group ID for reads; if blank, no consumer group is set", ) - testerCmd.Flags().StringVar( - &testerConfig.tlsCACert, - "tls-ca-cert", - "", - "Path to client CA cert PEM file if using TLS", - ) - testerCmd.Flags().StringVar( - &testerConfig.tlsCert, - "tls-cert", - "", - "Path to client cert PEM file if using TLS", - ) - testerCmd.Flags().BoolVar( - &testerConfig.tlsEnabled, - "tls-enabled", - false, - "Use TLS for communication with brokers", - ) - testerCmd.Flags().StringVar( - &testerConfig.tlsKey, - "tls-key", - "", - "Path to client private key PEM file if using TLS", - ) - testerCmd.Flags().StringVar( - &testerConfig.tlsServerName, - "tls-server-name", - "", - "Server name to use for TLS cert verification", - ) - testerCmd.Flags().BoolVar( - &testerConfig.tlsSkipVerify, - "tls-skip-verify", - false, - "Skip hostname verification when using TLS", - ) testerCmd.Flags().StringVar( &testerConfig.topic, "topic", @@ -108,24 +58,14 @@ func init() { 5, "Approximate number of messages to write per sec", ) - testerCmd.Flags().StringVarP( - &testerConfig.zkAddr, - "zk-addr", - "z", - "localhost:2181", - "Zookeeper address", - ) testerCmd.MarkFlagRequired("topic") - + addSharedFlags(testerCmd, &testerConfig.shared) RootCmd.AddCommand(testerCmd) } func testerPreRun(cmd *cobra.Command, args []string) error { - if testerConfig.zkAddr == "" && tailConfig.brokerAddr == "" { - return errors.New("Must set either broker-addr or zk-addr") - } - return nil + return testerConfig.shared.validate() } func testerRun(cmd *cobra.Command, args []string) error { @@ -150,10 +90,12 @@ func testerRun(cmd *cobra.Command, args []string) error { } func runTestReader(ctx context.Context) error { - connector, err := getConnector(ctx) + adminClient, err := testerConfig.shared.getAdminClient(ctx, nil, true) if err != nil { return err } + defer adminClient.Close() + connector := adminClient.GetConnector() log.Infof( "This will read test messages from the '%s' topic in %s using the consumer group ID '%s'", @@ -197,10 +139,12 @@ func runTestReader(ctx context.Context) error { } func runTestWriter(ctx context.Context) error { - connector, err := getConnector(ctx) + adminClient, err := testerConfig.shared.getAdminClient(ctx, nil, true) if err != nil { return err } + defer adminClient.Close() + connector := adminClient.GetConnector() log.Infof( "This will write test messages to the '%s' topic in %s at a rate of %d/sec.", @@ -255,34 +199,3 @@ func runTestWriter(ctx context.Context) error { } } } - -func getConnector(ctx context.Context) (*admin.Connector, error) { - if testerConfig.brokerAddr == "" { - adminClient, err := admin.NewZKAdminClient( - ctx, - admin.ZKAdminClientConfig{ - ZKAddrs: []string{testerConfig.zkAddr}, - }, - ) - if err != nil { - return nil, err - } - return adminClient.GetConnector(), nil - } else { - tlsEnabled := (testerConfig.tlsEnabled || - testerConfig.tlsCACert != "" || - testerConfig.tlsCert != "" || - testerConfig.tlsKey != "") - return admin.NewConnector( - admin.ConnectorConfig{ - BrokerAddr: testerConfig.brokerAddr, - TLSEnabled: tlsEnabled, - CACertPath: testerConfig.tlsCACert, - CertPath: testerConfig.tlsCert, - KeyPath: testerConfig.tlsKey, - ServerName: testerConfig.tlsServerName, - SkipVerify: testerConfig.tlsSkipVerify, - }, - ) - } -} diff --git a/docker-compose-sasl.yml b/docker-compose-sasl.yml new file mode 100644 index 00000000..309b9197 --- /dev/null +++ b/docker-compose-sasl.yml @@ -0,0 +1,44 @@ +# This config sets up a simple, single-node cluster that's equipped to use SASL. +# See examples/tls for the associated cluster configs and certs. +# +# To verify that TLS is working properly, try something like: +# +# openssl s_client -debug -connect localhost:9093 -CAfile ca.crt \ +# -cert server.crt -key server.key +version: '2' + +services: + zookeeper: + image: "wurstmeister/zookeeper:latest" + ports: + - "2181:2181" + + kafka1: + image: wurstmeister/kafka:2.12-2.4.1 + ports: + - "9094:9094" + - "9095:9095" + environment: + KAFKA_ADVERTISED_HOST_NAME: 127.0.0.1 + KAFKA_BROKER_RACK: zone1 + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_LISTENERS: SASL_PLAINTEXT://:9094,SASL_SSL://:9095 + KAFKA_ADVERTISED_LISTENERS: SASL_PLAINTEXT://127.0.0.1:9094,SASL_SSL://127.0.0.1:9095 + KAFKA_BROKER_ID: 1 + KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true" + KAFKA_SSL_KEYSTORE_LOCATION: "/certs/kafka.keystore.jks" + KAFKA_SSL_KEYSTORE_PASSWORD: "test123" + KAFKA_SSL_KEY_PASSWORD: "test123" + KAFKA_SSL_TRUSTSTORE_LOCATION: "/certs/kafka.truststore.jks" + KAFKA_SSL_TRUSTSTORE_PASSWORD: "test123" + KAFKA_SSL_CLIENT_AUTH: "none" + KAFKA_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: "" + KAFKA_SECURITY_INTER_BROKER_PROTOCOL: "SSL" + KAFKA_SASL_ENABLED_MECHANISMS: "PLAIN,SCRAM-SHA-256,SCRAM-SHA-512" + KAFKA_OPTS: "-Djava.security.auth.login.config=/opt/kafka/config/kafka_server_jaas.conf" + CUSTOM_INIT_SCRIPT: |- + echo -e 'KafkaServer {\norg.apache.kafka.common.security.scram.ScramLoginModule required\n username="adminscram"\n password="admin-secret";\n org.apache.kafka.common.security.plain.PlainLoginModule required\n username="adminplain"\n password="admin-secret"\n user_adminplain="admin-secret";\n };' > /opt/kafka/config/kafka_server_jaas.conf; + /opt/kafka/bin/kafka-configs.sh --zookeeper zookeeper:2181 --alter --add-config 'SCRAM-SHA-256=[password=admin-secret-256],SCRAM-SHA-512=[password=admin-secret-512]' --entity-type users --entity-name adminscram + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - ./examples/auth/certs:/certs diff --git a/docker-compose-tls.yml b/docker-compose-tls.yml index f7bb8596..318223a5 100644 --- a/docker-compose-tls.yml +++ b/docker-compose-tls.yml @@ -13,7 +13,7 @@ services: ports: - "2181:2181" - kafka: + kafka1: image: wurstmeister/kafka:2.12-2.4.1 ports: - "9092:9092" @@ -25,15 +25,15 @@ services: KAFKA_LISTENERS: PLAINTEXT://:9092,SSL://:9093 KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://127.0.0.1:9092,SSL://127.0.0.1:9093 KAFKA_BROKER_ID: 1 - KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true' - KAFKA_SSL_KEYSTORE_LOCATION: '/certs/kafka.keystore.jks' - KAFKA_SSL_KEYSTORE_PASSWORD: 'test123' - KAFKA_SSL_KEY_PASSWORD: 'test123' - KAFKA_SSL_TRUSTSTORE_LOCATION: '/certs/kafka.truststore.jks' - KAFKA_SSL_TRUSTSTORE_PASSWORD: 'test123' - KAFKA_SSL_CLIENT_AUTH: 'none' - KAFKA_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: '' - KAFKA_SECURITY_INTER_BROKER_PROTOCOL: 'SSL' + KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true" + KAFKA_SSL_KEYSTORE_LOCATION: "/certs/kafka.keystore.jks" + KAFKA_SSL_KEYSTORE_PASSWORD: "test123" + KAFKA_SSL_KEY_PASSWORD: "test123" + KAFKA_SSL_TRUSTSTORE_LOCATION: "/certs/kafka.truststore.jks" + KAFKA_SSL_TRUSTSTORE_PASSWORD: "test123" + KAFKA_SSL_CLIENT_AUTH: "none" + KAFKA_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: "" + KAFKA_SECURITY_INTER_BROKER_PROTOCOL: "SSL" volumes: - /var/run/docker.sock:/var/run/docker.sock - ./examples/auth/certs:/certs diff --git a/go.sum b/go.sum index 7cfc5394..8a630808 100644 --- a/go.sum +++ b/go.sum @@ -223,6 +223,7 @@ golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae h1:/WDfKMnPU+m5M4xB+6x4kaepxRw6jWvR5iDRdvjHgy8= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= diff --git a/pkg/admin/brokerclient.go b/pkg/admin/brokerclient.go index 60f767ae..f41266f4 100644 --- a/pkg/admin/brokerclient.go +++ b/pkg/admin/brokerclient.go @@ -42,6 +42,7 @@ func NewBrokerAdminClient( } client := connector.KafkaClient + log.Debugf("Getting supported API versions") apiVersions, err := client.ApiVersions(ctx, kafka.ApiVersionsRequest{}) if err != nil { return nil, err diff --git a/pkg/admin/connector.go b/pkg/admin/connector.go index f7b7b844..b185452d 100644 --- a/pkg/admin/connector.go +++ b/pkg/admin/connector.go @@ -5,15 +5,24 @@ import ( "crypto/x509" "fmt" "io/ioutil" + "strings" "time" "github.com/segmentio/kafka-go" + "github.com/segmentio/kafka-go/sasl" + "github.com/segmentio/kafka-go/sasl/plain" + "github.com/segmentio/kafka-go/sasl/scram" log "github.com/sirupsen/logrus" ) type ConnectorConfig struct { BrokerAddr string - TLSEnabled bool + TLS TLSConfig + SASL SASLConfig +} + +type TLSConfig struct { + Enabled bool CertPath string KeyPath string CACertPath string @@ -21,6 +30,13 @@ type ConnectorConfig struct { SkipVerify bool } +type SASLConfig struct { + Enabled bool + Mechanism string + Username string + Password string +} + type Connector struct { Config ConnectorConfig Dialer *kafka.Dialer @@ -32,54 +48,112 @@ func NewConnector(config ConnectorConfig) (*Connector, error) { Config: config, } + var saslMechanism sasl.Mechanism var tlsConfig *tls.Config + var err error - if !config.TLSEnabled { + if config.SASL.Enabled { + switch strings.ToLower(config.SASL.Mechanism) { + case "plain": + saslMechanism = plain.Mechanism{ + Username: config.SASL.Username, + Password: config.SASL.Password, + } + case "scram-sha-256": + saslMechanism, err = scram.Mechanism( + scram.SHA256, + config.SASL.Username, + config.SASL.Password, + ) + if err != nil { + return nil, err + } + case "scram-sha-512": + saslMechanism, err = scram.Mechanism( + scram.SHA512, + config.SASL.Username, + config.SASL.Password, + ) + if err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("Unrecognized SASL mechanism: %s", config.SASL.Mechanism) + } + } + + if !config.TLS.Enabled { connector.Dialer = kafka.DefaultDialer } else { var certs []tls.Certificate var caCertPool *x509.CertPool - if config.CertPath != "" && config.KeyPath != "" { - log.Debugf("Loading key pair from %s and %s", config.CertPath, config.KeyPath) - cert, err := tls.LoadX509KeyPair(config.CertPath, config.KeyPath) + if config.TLS.CertPath != "" && config.TLS.KeyPath != "" { + log.Debugf( + "Loading key pair from %s and %s", + config.TLS.CertPath, + config.TLS.KeyPath, + ) + cert, err := tls.LoadX509KeyPair(config.TLS.CertPath, config.TLS.KeyPath) if err != nil { return nil, err } certs = append(certs, cert) } - if config.CACertPath != "" { - log.Debugf("Adding CA certs from %s", config.CACertPath) + if config.TLS.CACertPath != "" { + log.Debugf("Adding CA certs from %s", config.TLS.CACertPath) caCertPool = x509.NewCertPool() - caCertContents, err := ioutil.ReadFile(config.CACertPath) + caCertContents, err := ioutil.ReadFile(config.TLS.CACertPath) if err != nil { return nil, err } if ok := caCertPool.AppendCertsFromPEM(caCertContents); !ok { - return nil, fmt.Errorf("Could not append CA certs from %s", config.CACertPath) + return nil, fmt.Errorf( + "Could not append CA certs from %s", + config.TLS.CACertPath, + ) } } tlsConfig = &tls.Config{ Certificates: certs, RootCAs: caCertPool, - InsecureSkipVerify: config.SkipVerify, - ServerName: config.ServerName, + InsecureSkipVerify: config.TLS.SkipVerify, + ServerName: config.TLS.ServerName, } connector.Dialer = &kafka.Dialer{ - Timeout: 10 * time.Second, - TLS: tlsConfig, + SASLMechanism: saslMechanism, + Timeout: 10 * time.Second, + TLS: tlsConfig, } } + log.Debugf("Connecting to cluster on address %s with TLS enabled=%s, SASL enabled=%s", + config.BrokerAddr, + config.TLS.Enabled, + config.SASL.Enabled, + ) connector.KafkaClient = &kafka.Client{ Addr: kafka.TCP(config.BrokerAddr), Transport: &kafka.Transport{ Dial: connector.Dialer.DialFunc, + SASL: saslMechanism, TLS: tlsConfig, }, } return connector, nil } + +func ValidateSASLMechanism(mechanism string) error { + switch strings.ToLower(mechanism) { + case "plain", "scram-sha-256", "scram-sha-512": + return nil + default: + return fmt.Errorf( + "SASL mechanism '%s' is not valid; choices are PLAIN, SCRAM-SHA-256, and SCRAM-SHA-512", + mechanism, + ) + } +} diff --git a/pkg/config/cluster.go b/pkg/config/cluster.go index ce2af374..a608e385 100644 --- a/pkg/config/cluster.go +++ b/pkg/config/cluster.go @@ -83,6 +83,10 @@ type ClusterSpec struct { // TLS stores how we should use TLS with broker connections, if appropriate. Only // applies if using the broker admin. TLS TLSConfig `json:"tls"` + + // SASL stores how we should use SASL with broker connections, if appropriate. Only + // applies if using the broker admin. + SASL SASLConfig `json:"sasl"` } type TLSConfig struct { @@ -94,6 +98,13 @@ type TLSConfig struct { SkipVerify bool `json:"skipVerify"` } +type SASLConfig struct { + Enabled bool `json:"enabled"` + Mechanism string `json:"mechanism"` + Username string `json:"username"` + Password string `json:"password"` +} + // Validate evaluates whether the cluster config is valid. func (c ClusterConfig) Validate() error { var err error @@ -133,6 +144,18 @@ func (c ClusterConfig) Validate() error { errors.New("TLS not supported with zk access mode; omit zk addresses to fix"), ) } + if c.Spec.SASL.Enabled && len(c.Spec.ZKAddrs) > 0 { + err = multierror.Append( + err, + errors.New("SASL not supported with zk access mode; omit zk addresses to fix"), + ) + } + + if c.Spec.SASL.Enabled { + if saslErr := admin.ValidateSASLMechanism(c.Spec.SASL.Mechanism); saslErr != nil { + err = multierror.Append(err, saslErr) + } + } return err } @@ -158,12 +181,19 @@ func (c ClusterConfig) NewAdminClient( admin.BrokerAdminClientConfig{ ConnectorConfig: admin.ConnectorConfig{ BrokerAddr: c.Spec.BootstrapAddrs[0], - TLSEnabled: c.Spec.TLS.Enabled, - CACertPath: c.absPath(c.Spec.TLS.CACertPath), - CertPath: c.absPath(c.Spec.TLS.CertPath), - KeyPath: c.absPath(c.Spec.TLS.KeyPath), - ServerName: c.Spec.TLS.ServerName, - SkipVerify: c.Spec.TLS.SkipVerify, + TLS: admin.TLSConfig{ + Enabled: c.Spec.TLS.Enabled, + CACertPath: c.absPath(c.Spec.TLS.CACertPath), + KeyPath: c.absPath(c.Spec.TLS.KeyPath), + ServerName: c.Spec.TLS.ServerName, + SkipVerify: c.Spec.TLS.SkipVerify, + }, + SASL: admin.SASLConfig{ + Enabled: c.Spec.SASL.Enabled, + Mechanism: c.Spec.SASL.Mechanism, + Username: c.Spec.SASL.Username, + Password: c.Spec.SASL.Password, + }, }, ReadOnly: readOnly, }, From 06e4bc9e37a3bf538876d658a030a216dd4e13a6 Mon Sep 17 00:00:00 2001 From: Benjamin Yolken Date: Wed, 25 Nov 2020 17:24:46 -0800 Subject: [PATCH 13/19] Allow overriding SASL username and password --- cmd/topicctl/subcmd/apply.go | 23 +++++++++-------- cmd/topicctl/subcmd/bootstrap.go | 22 ++++++++-------- cmd/topicctl/subcmd/check.go | 28 +++++++++++---------- cmd/topicctl/subcmd/get.go | 3 ++- cmd/topicctl/subcmd/shared.go | 43 +++++++++++++++++++++----------- pkg/admin/connector.go | 2 +- pkg/apply/apply_test.go | 4 +-- pkg/check/check_test.go | 2 +- pkg/config/cluster.go | 23 +++++++++++++++-- 9 files changed, 94 insertions(+), 56 deletions(-) diff --git a/cmd/topicctl/subcmd/apply.go b/cmd/topicctl/subcmd/apply.go index 96139b1c..cfba8749 100644 --- a/cmd/topicctl/subcmd/apply.go +++ b/cmd/topicctl/subcmd/apply.go @@ -28,7 +28,6 @@ var applyCmd = &cobra.Command{ type applyCmdConfig struct { brokersToRemove []int brokerThrottleMBsOverride int - clusterConfig string dryRun bool partitionBatchSizeOverride int pathPrefix string @@ -37,6 +36,8 @@ type applyCmdConfig struct { skipConfirm bool sleepLoopDuration time.Duration + shared sharedOptions + retentionDropStepDuration time.Duration } @@ -55,12 +56,6 @@ func init() { 0, "Broker throttle override (MB/sec)", ) - applyCmd.Flags().StringVar( - &applyConfig.clusterConfig, - "cluster-config", - os.Getenv("TOPICCTL_CLUSTER_CONFIG"), - "Cluster config path", - ) applyCmd.Flags().BoolVar( &applyConfig.dryRun, "dry-run", @@ -104,7 +99,7 @@ func init() { "Amount of time to wait between partition checks", ) - applyCmd.MarkFlagRequired("cluster-config") + addSharedConfigOnlyFlags(applyCmd, &applyConfig.shared) RootCmd.AddCommand(applyCmd) } @@ -192,7 +187,13 @@ func applyTopic( adminClient, ok := adminClients[clusterConfigPath] if !ok { - adminClient, err = clusterConfig.NewAdminClient(ctx, nil, applyConfig.dryRun) + adminClient, err = clusterConfig.NewAdminClient( + ctx, + nil, + applyConfig.dryRun, + applyConfig.shared.saslUsername, + applyConfig.shared.saslPassword, + ) if err != nil { return err } @@ -232,8 +233,8 @@ func applyTopic( } func clusterConfigForTopicApply(topicConfigPath string) (string, error) { - if applyConfig.clusterConfig != "" { - return applyConfig.clusterConfig, nil + if applyConfig.shared.clusterConfig != "" { + return applyConfig.shared.clusterConfig, nil } return filepath.Abs( diff --git a/cmd/topicctl/subcmd/bootstrap.go b/cmd/topicctl/subcmd/bootstrap.go index cc0fe14a..6110d5ae 100644 --- a/cmd/topicctl/subcmd/bootstrap.go +++ b/cmd/topicctl/subcmd/bootstrap.go @@ -2,7 +2,6 @@ package subcmd import ( "context" - "os" "github.com/segmentio/topicctl/pkg/cli" "github.com/segmentio/topicctl/pkg/config" @@ -17,22 +16,17 @@ var bootstrapCmd = &cobra.Command{ } type bootstrapCmdConfig struct { - clusterConfig string matchRegexp string excludeRegexp string outputDir string overwrite bool + + shared sharedOptions } var bootstrapConfig bootstrapCmdConfig func init() { - bootstrapCmd.Flags().StringVar( - &bootstrapConfig.clusterConfig, - "cluster-config", - os.Getenv("TOPICCTL_CLUSTER_CONFIG"), - "Cluster config", - ) bootstrapCmd.Flags().StringVar( &bootstrapConfig.matchRegexp, "match", @@ -59,8 +53,8 @@ func init() { "Overwrite existing configs in output directory", ) + addSharedConfigOnlyFlags(bootstrapCmd, &bootstrapConfig.shared) bootstrapCmd.MarkFlagRequired("cluster-config") - RootCmd.AddCommand(bootstrapCmd) } @@ -68,11 +62,17 @@ func bootstrapRun(cmd *cobra.Command, args []string) error { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - clusterConfig, err := config.LoadClusterFile(bootstrapConfig.clusterConfig) + clusterConfig, err := config.LoadClusterFile(bootstrapConfig.shared.clusterConfig) if err != nil { return err } - adminClient, err := clusterConfig.NewAdminClient(ctx, nil, true) + adminClient, err := clusterConfig.NewAdminClient( + ctx, + nil, + true, + bootstrapConfig.shared.saslUsername, + bootstrapConfig.shared.saslPassword, + ) if err != nil { return err } diff --git a/cmd/topicctl/subcmd/check.go b/cmd/topicctl/subcmd/check.go index 48de5c3e..33f2aaf1 100644 --- a/cmd/topicctl/subcmd/check.go +++ b/cmd/topicctl/subcmd/check.go @@ -21,21 +21,16 @@ var checkCmd = &cobra.Command{ } type checkCmdConfig struct { - clusterConfig string - checkLeaders bool - pathPrefix string - validateOnly bool + checkLeaders bool + pathPrefix string + validateOnly bool + + shared sharedOptions } var checkConfig checkCmdConfig func init() { - checkCmd.Flags().StringVar( - &checkConfig.clusterConfig, - "cluster-config", - os.Getenv("TOPICCTL_CLUSTER_CONFIG"), - "Cluster config", - ) checkCmd.Flags().StringVar( &checkConfig.pathPrefix, "path-prefix", @@ -55,6 +50,7 @@ func init() { "Validate configs only, without connecting to cluster", ) + addSharedConfigOnlyFlags(checkCmd, &checkConfig.shared) RootCmd.AddCommand(checkCmd) } @@ -136,7 +132,13 @@ func checkTopicFile( var ok bool adminClient, ok = adminClients[clusterConfigPath] if !ok { - adminClient, err = clusterConfig.NewAdminClient(ctx, nil, true) + adminClient, err = clusterConfig.NewAdminClient( + ctx, + nil, + true, + checkConfig.shared.saslUsername, + checkConfig.shared.saslPassword, + ) if err != nil { return false, err } @@ -177,8 +179,8 @@ func checkTopicFile( } func clusterConfigForTopicCheck(topicConfigPath string) (string, error) { - if checkConfig.clusterConfig != "" { - return checkConfig.clusterConfig, nil + if checkConfig.shared.clusterConfig != "" { + return checkConfig.shared.clusterConfig, nil } return filepath.Abs( diff --git a/cmd/topicctl/subcmd/get.go b/cmd/topicctl/subcmd/get.go index 69c2ffd9..bd2d7ab3 100644 --- a/cmd/topicctl/subcmd/get.go +++ b/cmd/topicctl/subcmd/get.go @@ -29,7 +29,8 @@ var getCmd = &cobra.Command{ } type getCmdConfig struct { - full bool + full bool + shared sharedOptions } diff --git a/cmd/topicctl/subcmd/shared.go b/cmd/topicctl/subcmd/shared.go index 6c8f2ee6..7122da93 100644 --- a/cmd/topicctl/subcmd/shared.go +++ b/cmd/topicctl/subcmd/shared.go @@ -80,7 +80,13 @@ func (s sharedOptions) getAdminClient( if err != nil { return nil, err } - return clusterConfig.NewAdminClient(ctx, sess, true) + return clusterConfig.NewAdminClient( + ctx, + sess, + true, + s.saslUsername, + s.saslPassword, + ) } else if s.brokerAddr != "" { tlsEnabled := (s.tlsEnabled || s.tlsCACert != "" || @@ -145,19 +151,7 @@ func addSharedFlags(cmd *cobra.Command, options *sharedOptions) { &options.saslMechanism, "sasl-mechanism", "", - "SASL mechanism if using SASL (either: plain, scram-sha-256, or scram-sha-512)", - ) - cmd.Flags().StringVar( - &options.saslPassword, - "sasl-password", - os.Getenv("TOPICCTL_SASL_PASSWORD"), - "SASL password if using SASL", - ) - cmd.Flags().StringVar( - &options.saslUsername, - "sasl-username", - os.Getenv("TOPICCTL_SASL_USERNAME"), - "SASL username if using SASL", + "SASL mechanism if using SASL (choices: PLAIN, SCRAM-SHA-256, or SCRAM-SHA-512)", ) cmd.Flags().StringVar( &options.tlsCACert, @@ -209,3 +203,24 @@ func addSharedFlags(cmd *cobra.Command, options *sharedOptions) { "Prefix for cluster-related nodes in zk", ) } + +func addSharedConfigOnlyFlags(cmd *cobra.Command, options *sharedOptions) { + cmd.Flags().StringVar( + &options.clusterConfig, + "cluster-config", + os.Getenv("TOPICCTL_CLUSTER_CONFIG"), + "Cluster config", + ) + cmd.Flags().StringVar( + &options.saslPassword, + "sasl-password", + os.Getenv("TOPICCTL_SASL_PASSWORD"), + "SASL password if using SASL; will override value set in cluster config", + ) + cmd.Flags().StringVar( + &options.saslUsername, + "sasl-username", + os.Getenv("TOPICCTL_SASL_USERNAME"), + "SASL username if using SASL; will override value set in cluster config", + ) +} diff --git a/pkg/admin/connector.go b/pkg/admin/connector.go index b185452d..a316656b 100644 --- a/pkg/admin/connector.go +++ b/pkg/admin/connector.go @@ -129,7 +129,7 @@ func NewConnector(config ConnectorConfig) (*Connector, error) { } } - log.Debugf("Connecting to cluster on address %s with TLS enabled=%s, SASL enabled=%s", + log.Debugf("Connecting to cluster on address %s with TLS enabled=%v, SASL enabled=%v", config.BrokerAddr, config.TLS.Enabled, config.SASL.Enabled, diff --git a/pkg/apply/apply_test.go b/pkg/apply/apply_test.go index a14b4d3d..3851bfcc 100644 --- a/pkg/apply/apply_test.go +++ b/pkg/apply/apply_test.go @@ -879,7 +879,7 @@ func TestApplyOverrides(t *testing.T) { }, } - adminClient, err := clusterConfig.NewAdminClient(ctx, nil, false) + adminClient, err := clusterConfig.NewAdminClient(ctx, nil, false, "", "") require.NoError(t, err) applier, err := NewTopicApplier( @@ -922,7 +922,7 @@ func testApplier( }, } - adminClient, err := clusterConfig.NewAdminClient(ctx, nil, false) + adminClient, err := clusterConfig.NewAdminClient(ctx, nil, false, "", "") require.NoError(t, err) applier, err := NewTopicApplier( diff --git a/pkg/check/check_test.go b/pkg/check/check_test.go index 7f066acb..592d127e 100644 --- a/pkg/check/check_test.go +++ b/pkg/check/check_test.go @@ -28,7 +28,7 @@ func TestCheck(t *testing.T) { }, } - adminClient, err := clusterConfig.NewAdminClient(ctx, nil, false) + adminClient, err := clusterConfig.NewAdminClient(ctx, nil, false, "", "") require.NoError(t, err) topicName := util.RandomString("check-topic-", 6) diff --git a/pkg/config/cluster.go b/pkg/config/cluster.go index a608e385..28f28ce0 100644 --- a/pkg/config/cluster.go +++ b/pkg/config/cluster.go @@ -173,9 +173,28 @@ func (c ClusterConfig) NewAdminClient( ctx context.Context, sess *session.Session, readOnly bool, + usernameOverride string, + passwordOverride string, ) (admin.Client, error) { if len(c.Spec.ZKAddrs) == 0 { log.Debug("No ZK addresses provided, using broker admin client") + + var saslUsername string + var saslPassword string + if usernameOverride != "" { + log.Debugf("Setting SASL username from override value") + saslUsername = usernameOverride + } else { + saslUsername = c.Spec.SASL.Username + } + + if passwordOverride != "" { + log.Debugf("Setting SASL password from override value") + saslPassword = passwordOverride + } else { + saslPassword = c.Spec.SASL.Password + } + return admin.NewBrokerAdminClient( ctx, admin.BrokerAdminClientConfig{ @@ -191,8 +210,8 @@ func (c ClusterConfig) NewAdminClient( SASL: admin.SASLConfig{ Enabled: c.Spec.SASL.Enabled, Mechanism: c.Spec.SASL.Mechanism, - Username: c.Spec.SASL.Username, - Password: c.Spec.SASL.Password, + Username: saslUsername, + Password: saslPassword, }, }, ReadOnly: readOnly, From 8ace4da74da4b57e0ffdae8015b5b4d11c297799 Mon Sep 17 00:00:00 2001 From: Benjamin Yolken Date: Wed, 25 Nov 2020 18:45:58 -0800 Subject: [PATCH 14/19] Update README and examples --- README.md | 63 +++++++++++++------ cmd/topicctl/subcmd/shared.go | 12 ++++ ...ompose-sasl.yml => docker-compose-auth.yml | 50 ++++++++------- docker-compose-tls.yml | 39 ------------ examples/auth/cluster.yaml | 20 ++++-- examples/auth/topics/topic-default.yaml | 2 +- examples/local-cluster/cluster.yaml | 14 +++-- pkg/config/cluster.go | 20 ------ pkg/config/cluster_test.go | 7 +-- pkg/config/load_test.go | 1 - .../test-cluster/cluster-invalid.yaml | 5 +- pkg/config/testdata/test-cluster/cluster.yaml | 1 - 12 files changed, 114 insertions(+), 120 deletions(-) rename docker-compose-sasl.yml => docker-compose-auth.yml (54%) delete mode 100644 docker-compose-tls.yml diff --git a/README.md b/README.md index e4289cef..1814e126 100644 --- a/README.md +++ b/README.md @@ -23,12 +23,6 @@ non-Kafka-related contexts. See [this blog post](https://segment.com/blog/easier-management-of-Kafka-topics-with-topicctl/) for more details. -## Roadmap - -We are in the process of making some changes to (optionally) remove the ZK dependency and also to -support some additional security features like TLS. See -[this page](https://github.com/segmentio/topicctl/wiki/v1-Plan) for the current plan and status. - ## Getting started ### Installation @@ -69,7 +63,7 @@ topicctl apply --skip-confirm examples/local-cluster/topics/*yaml 4. Send some test messages to the `topic-default` topic: ``` -topicctl tester --zk-addr=localhost:2181 --topic=topic-default +topicctl tester --broker-addr=localhost:2181 --topic=topic-default ``` 5. Open up the repl (while keeping the tester running in a separate terminal): @@ -223,9 +217,9 @@ typically source-controlled so that changes can be reviewed before being applied ### Clusters -Each cluster associated with a managed topic must have a config. These -configs can also be used with the `get`, `repl`, and `tail` subcommands instead -of specifying a ZooKeeper address. +Each cluster associated with a managed topic must have a config. These configs can also be used +with the `get`, `repl`, `reset-offsets`, and `tail` subcommands instead of specifying a broker or +ZooKeeper address. The following shows an annotated example: @@ -238,16 +232,30 @@ meta: Test cluster for topicctl. spec: - versionMajor: v0.10 # Version bootstrapAddrs: # One or more broker bootstrap addresses - my-cluster.example.com:9092 + clusterID: abc-123-xyz # Expected cluster ID for cluster (optional, used as safety check only) + + # ZooKeeper access settings (required for pre-v2 clusters) zkAddrs: # One or more cluster zookeeper addresses; if these are - zk.example.com:2181 # omitted, then the cluster will only be accessed via broker APIs; # see the section below on cluster access for more details. - zkPrefix: my-cluster # Prefix for zookeeper nodes + zkPrefix: my-cluster # Prefix for zookeeper nodes if using zookeeper access zkLockPath: /topicctl/locks # Path used for apply locks (optional) - clusterID: abc-123-xyz # Expected cluster ID for cluster (optional, used as - # safety check only) + + # TLS/SSL settings (optional, not supported if using ZooKeeper) + tls: + enabled: true # Whether TLS is enabled + caCertPath: path/to/ca.crt # Path to CA cert to be used (optional) + certPath: path/to/client.crt # Path to client cert to be used (optional) + keyPath: path/to/client.key # Path to client key to be used (optional) + + # SASL settings (optional, not supported if using ZooKeeper) + sasl: + enabled: true # Whether SASL is enabled + mechanism: SCRAM-SHA-512 # Mechanism to use; choices are PLAIN, SCRAM-SHA-256, and SCRAM-SHA-512 + username: my-username # Username; can also be set via TOPICCTL_SASL_USERNAME environment variable + password: my-password # Password; can also be set via TOPICCTL_SASL_PASSWORD environment variable ``` Note that the `name`, `environment`, `region`, and `description` fields are used @@ -357,7 +365,7 @@ The `apply` subcommand can make changes, but under the following conditions: 7. Partition replica migrations are protected via ["throttles"](https://kafka.apache.org/0101/documentation.html#rep-throttle) to prevent the cluster network from getting overwhelmed -8. Before applying, the tool checks the cluster ID in ZooKeeper against the expected value in the +8. Before applying, the tool checks the cluster ID against the expected value in the cluster config. This can help prevent errors around applying in the wrong cluster when multiple clusters are accessed through the same address, e.g `localhost:2181`. @@ -403,17 +411,32 @@ operations, >= 2.4 for applies). ### TLS -TLS is supported when running `topicctl` in the exclusive broker API mode. To use this, either -set `--tls-enabled` in the command-line or, if using a cluster config, set `tlsEnabled: true` -in the `clientAuth` section of the latter. +TLS (referred to by the older name "SSL" in the Kafka documentation) is supported when running +`topicctl` in the exclusive broker API mode. To use this, either set `--tls-enabled` in the +command-line or, if using a cluster config, set `enabled: true` in the `TLS` section of +the latter. In addition to standard TLS, the tool also supports mutual TLS using custom certs, keys, and CA certs (in PEM format). As with the enabling of TLS, these can be configured either on the -command-line or in a cluster config. See [this config](examples/tls/cluster.yaml) for an example. +command-line or in a cluster config. See [this config](examples/auth/cluster.yaml) for an example. ### SASL -Coming soon. +`topicctl` supports SASL authentication when running in the exclusive broker API mode. To use this, +either set the `--sasl-mechanism`, `--sasl-username`, and `--sasl-password` flags on the command +line or fill out the `SASL` section of the cluster config. + +If using the cluster config, the username and password can still be set on the command-line +or via the `TOPICCTL_SASL_USERNAME` and `TOPICCTL_SASL_PASSWORD` environment variables. + +The tool currently supports the following SASL mechanisms: + +1. `PLAIN` +2. `SCRAM-SHA-256` +3. `SCRAM-SHA-512` + +Note that SASL can be run either with or without TLS, although the former is generally more +secure. ## Development diff --git a/cmd/topicctl/subcmd/shared.go b/cmd/topicctl/subcmd/shared.go index 7122da93..004c7456 100644 --- a/cmd/topicctl/subcmd/shared.go +++ b/cmd/topicctl/subcmd/shared.go @@ -153,6 +153,18 @@ func addSharedFlags(cmd *cobra.Command, options *sharedOptions) { "", "SASL mechanism if using SASL (choices: PLAIN, SCRAM-SHA-256, or SCRAM-SHA-512)", ) + cmd.Flags().StringVar( + &options.saslPassword, + "sasl-password", + os.Getenv("TOPICCTL_SASL_PASSWORD"), + "SASL password if using SASL; will override value set in cluster config", + ) + cmd.Flags().StringVar( + &options.saslUsername, + "sasl-username", + os.Getenv("TOPICCTL_SASL_USERNAME"), + "SASL username if using SASL; will override value set in cluster config", + ) cmd.Flags().StringVar( &options.tlsCACert, "tls-ca-cert", diff --git a/docker-compose-sasl.yml b/docker-compose-auth.yml similarity index 54% rename from docker-compose-sasl.yml rename to docker-compose-auth.yml index 309b9197..16b70690 100644 --- a/docker-compose-sasl.yml +++ b/docker-compose-auth.yml @@ -1,10 +1,12 @@ -# This config sets up a simple, single-node cluster that's equipped to use SASL. -# See examples/tls for the associated cluster configs and certs. +# This config sets up a simple, single-node cluster that's equipped to use SSL/TLS and/or SASL. +# It exposes access on four separate ports: # -# To verify that TLS is working properly, try something like: +# 1. 9092: plaintext, no SASL +# 2. 9093: SSL, no SASL +# 3. 9094: SASL over plaintext +# 4. 9095: SASL over SSL # -# openssl s_client -debug -connect localhost:9093 -CAfile ca.crt \ -# -cert server.crt -key server.key +# See examples/auth for the associated cluster configs and certs. version: '2' services: @@ -13,28 +15,32 @@ services: ports: - "2181:2181" - kafka1: + kafka: image: wurstmeister/kafka:2.12-2.4.1 + restart: on-failure:3 + links: + - zookeeper ports: - - "9094:9094" - - "9095:9095" + - 9092:9092 + - 9093:9093 + - 9094:9094 + - 9095:9095 environment: - KAFKA_ADVERTISED_HOST_NAME: 127.0.0.1 - KAFKA_BROKER_RACK: zone1 - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - KAFKA_LISTENERS: SASL_PLAINTEXT://:9094,SASL_SSL://:9095 - KAFKA_ADVERTISED_LISTENERS: SASL_PLAINTEXT://127.0.0.1:9094,SASL_SSL://127.0.0.1:9095 KAFKA_BROKER_ID: 1 - KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true" - KAFKA_SSL_KEYSTORE_LOCATION: "/certs/kafka.keystore.jks" - KAFKA_SSL_KEYSTORE_PASSWORD: "test123" - KAFKA_SSL_KEY_PASSWORD: "test123" - KAFKA_SSL_TRUSTSTORE_LOCATION: "/certs/kafka.truststore.jks" - KAFKA_SSL_TRUSTSTORE_PASSWORD: "test123" - KAFKA_SSL_CLIENT_AUTH: "none" - KAFKA_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: "" - KAFKA_SECURITY_INTER_BROKER_PROTOCOL: "SSL" + KAFKA_ADVERTISED_HOST_NAME: localhost + KAFKA_ADVERTISED_PORT: 9092 + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_MESSAGE_MAX_BYTES: 200000000 + KAFKA_LISTENERS: "PLAINTEXT://:9092,SSL://:9093,SASL_PLAINTEXT://:9094,SASL_SSL://:9095" + KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://localhost:9092,SSL://localhost:9093,SASL_PLAINTEXT://localhost:9094,SASL_SSL://localhost:9095" KAFKA_SASL_ENABLED_MECHANISMS: "PLAIN,SCRAM-SHA-256,SCRAM-SHA-512" + KAFKA_SSL_KEYSTORE_LOCATION: /certs/kafka.keystore.jks + KAFKA_SSL_KEYSTORE_PASSWORD: test123 + KAFKA_SSL_KEY_PASSWORD: test123 + KAFKA_SSL_TRUSTSTORE_LOCATION: /certs/kafka.truststore.jks + KAFKA_SSL_TRUSTSTORE_PASSWORD: test123 + KAFKA_SSL_CLIENT_AUTH: none + KAFKA_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: "" KAFKA_OPTS: "-Djava.security.auth.login.config=/opt/kafka/config/kafka_server_jaas.conf" CUSTOM_INIT_SCRIPT: |- echo -e 'KafkaServer {\norg.apache.kafka.common.security.scram.ScramLoginModule required\n username="adminscram"\n password="admin-secret";\n org.apache.kafka.common.security.plain.PlainLoginModule required\n username="adminplain"\n password="admin-secret"\n user_adminplain="admin-secret";\n };' > /opt/kafka/config/kafka_server_jaas.conf; diff --git a/docker-compose-tls.yml b/docker-compose-tls.yml deleted file mode 100644 index 318223a5..00000000 --- a/docker-compose-tls.yml +++ /dev/null @@ -1,39 +0,0 @@ -# This config sets up a simple, single-node cluster that's equipped to use TLS. -# See examples/tls for the associated cluster configs and certs. -# -# To verify that TLS is working properly, try something like: -# -# openssl s_client -debug -connect localhost:9093 -CAfile ca.crt \ -# -cert server.crt -key server.key -version: '2' - -services: - zookeeper: - image: "wurstmeister/zookeeper:latest" - ports: - - "2181:2181" - - kafka1: - image: wurstmeister/kafka:2.12-2.4.1 - ports: - - "9092:9092" - - "9093:9093" - environment: - KAFKA_ADVERTISED_HOST_NAME: 127.0.0.1 - KAFKA_BROKER_RACK: zone1 - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - KAFKA_LISTENERS: PLAINTEXT://:9092,SSL://:9093 - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://127.0.0.1:9092,SSL://127.0.0.1:9093 - KAFKA_BROKER_ID: 1 - KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true" - KAFKA_SSL_KEYSTORE_LOCATION: "/certs/kafka.keystore.jks" - KAFKA_SSL_KEYSTORE_PASSWORD: "test123" - KAFKA_SSL_KEY_PASSWORD: "test123" - KAFKA_SSL_TRUSTSTORE_LOCATION: "/certs/kafka.truststore.jks" - KAFKA_SSL_TRUSTSTORE_PASSWORD: "test123" - KAFKA_SSL_CLIENT_AUTH: "none" - KAFKA_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: "" - KAFKA_SECURITY_INTER_BROKER_PROTOCOL: "SSL" - volumes: - - /var/run/docker.sock:/var/run/docker.sock - - ./examples/auth/certs:/certs diff --git a/examples/auth/cluster.yaml b/examples/auth/cluster.yaml index b38bbb80..07b097c9 100644 --- a/examples/auth/cluster.yaml +++ b/examples/auth/cluster.yaml @@ -1,17 +1,29 @@ meta: - name: local-cluster-tls + name: local-cluster-auth environment: local-env region: local-region description: | - Test cluster + Test cluster that uses SSL/TLS and SASL to securely connect to brokers. Can be run + against compose setup defined in docker-compose-auth.yml in the repo root. spec: - versionMajor: v2 bootstrapAddrs: - - localhost:9093 + # To use just TLS without SASL, switch to port 9093 and disable SASL in the config below. + # To use just SASL without TLS, switch to port 9094 and disabled TLS in the config below. + - localhost:9095 tls: enabled: true caCertPath: certs/ca.crt certPath: certs/client.crt keyPath: certs/client.key skipVerify: true + sasl: + enabled: true + mechanism: SCRAM-SHA-512 + + # As an alternative to storing these in the config (probably not super-secure), these can be + # set by using the --sasl-username and --sasl-password flags or the + # TOPICCTL_SASL_USERNAME and TOPICCTL_SASL_PASSWORD environment variables when running + # topicctl. + username: adminscram + password: admin-secret-512 diff --git a/examples/auth/topics/topic-default.yaml b/examples/auth/topics/topic-default.yaml index aecafe68..d2555bd8 100644 --- a/examples/auth/topics/topic-default.yaml +++ b/examples/auth/topics/topic-default.yaml @@ -1,6 +1,6 @@ meta: name: topic-default - cluster: local-cluster-tls + cluster: local-cluster-auth environment: local-env region: local-region description: | diff --git a/examples/local-cluster/cluster.yaml b/examples/local-cluster/cluster.yaml index 812a6d57..ca142900 100644 --- a/examples/local-cluster/cluster.yaml +++ b/examples/local-cluster/cluster.yaml @@ -3,12 +3,16 @@ meta: environment: local-env region: local-region description: | - Test cluster + Test cluster that uses plaintext access to brokers. Can be run against compose setup defined + in docker-compose.yml in the repo root. spec: - versionMajor: v2 bootstrapAddrs: - localhost:9092 - zkAddrs: - - localhost:2181 - zkLockPath: /topicctl/locks + + # Uncomment these lines to access cluster via ZooKeeper instead of broker APIs (required + # for older cluster versions). + # + # zkAddrs: + # - localhost:2181 + # zkLockPath: /topicctl/locks diff --git a/pkg/config/cluster.go b/pkg/config/cluster.go index 28f28ce0..c307402c 100644 --- a/pkg/config/cluster.go +++ b/pkg/config/cluster.go @@ -13,17 +13,6 @@ import ( log "github.com/sirupsen/logrus" ) -// KafkaVersionMajor is a string type for storing Kafka versions. -type KafkaVersionMajor string - -const ( - // KafkaVersionMajor010 represents kafka v0.10 and its associated minor versions. - KafkaVersionMajor010 KafkaVersionMajor = "v0.10" - - // KafkaVersionMajor2 represents kafka v2 and its associated minor versions. - KafkaVersionMajor2 KafkaVersionMajor = "v2" -) - // ClusterConfig stores information about a cluster that's referred to by one // or more topic configs. These configs should reflect the reality of what's been // set up externally; there's no way to "apply" these at the moment. @@ -67,11 +56,6 @@ type ClusterSpec struct { // this check isn't done. ClusterID string `json:"clusterID"` - // VersionMajor stores the major version of the cluster. This isn't currently - // used for any logic in the tool, but it may be used in the future to adjust API calls - // and/or decide whether to use zk or brokers for certain information. - VersionMajor KafkaVersionMajor `json:"versionMajor"` - // DefaultThrottleMB is the default broker throttle used for migrations in this // cluster. If unset, then a reasonable default is used instead. DefaultThrottleMB int64 `json:"defaultThrottleMB"` @@ -125,10 +109,6 @@ func (c ClusterConfig) Validate() error { errors.New("At least one bootstrap broker address must be set"), ) } - if c.Spec.VersionMajor != KafkaVersionMajor010 && - c.Spec.VersionMajor != KafkaVersionMajor2 { - multierror.Append(err, errors.New("MajorVersion must be v0.10 or v2")) - } _, parseErr := c.GetDefaultRetentionDropStepDuration() if parseErr != nil { diff --git a/pkg/config/cluster_test.go b/pkg/config/cluster_test.go index db5bfee7..c0987983 100644 --- a/pkg/config/cluster_test.go +++ b/pkg/config/cluster_test.go @@ -26,7 +26,6 @@ func TestClusterValidate(t *testing.T) { Spec: ClusterSpec{ BootstrapAddrs: []string{"broker-addr"}, ZKAddrs: []string{"zk-addr"}, - VersionMajor: "v2", DefaultRetentionDropStepDurationStr: "5m", }, }, @@ -42,7 +41,6 @@ func TestClusterValidate(t *testing.T) { Spec: ClusterSpec{ BootstrapAddrs: []string{"broker-addr"}, ZKAddrs: []string{"zk-addr"}, - VersionMajor: "v2", }, }, expError: true, @@ -57,8 +55,7 @@ func TestClusterValidate(t *testing.T) { Description: "test-description", }, Spec: ClusterSpec{ - ZKAddrs: []string{"zk-addr"}, - VersionMajor: "v2", + ZKAddrs: []string{"zk-addr"}, }, }, expError: true, @@ -74,7 +71,6 @@ func TestClusterValidate(t *testing.T) { }, Spec: ClusterSpec{ BootstrapAddrs: []string{"broker-addr"}, - VersionMajor: "v2", }, }, expError: false, @@ -91,7 +87,6 @@ func TestClusterValidate(t *testing.T) { Spec: ClusterSpec{ BootstrapAddrs: []string{"broker-addr"}, ZKAddrs: []string{"zk-addr"}, - VersionMajor: "v2", DefaultRetentionDropStepDurationStr: "10xxx", }, }, diff --git a/pkg/config/load_test.go b/pkg/config/load_test.go index 16ed7de7..233d51b9 100644 --- a/pkg/config/load_test.go +++ b/pkg/config/load_test.go @@ -24,7 +24,6 @@ func TestLoadCluster(t *testing.T) { Description: "Test cluster\n", }, Spec: ClusterSpec{ - VersionMajor: KafkaVersionMajor010, BootstrapAddrs: []string{ "bootstrap-addr:9092", }, diff --git a/pkg/config/testdata/test-cluster/cluster-invalid.yaml b/pkg/config/testdata/test-cluster/cluster-invalid.yaml index 541353f5..10192341 100644 --- a/pkg/config/testdata/test-cluster/cluster-invalid.yaml +++ b/pkg/config/testdata/test-cluster/cluster-invalid.yaml @@ -7,4 +7,7 @@ meta: clusterId: test-cluster-id spec: - versionMajor: v0.40 + zkAddrs: + - localhost:2181 + tls: + enabled: true diff --git a/pkg/config/testdata/test-cluster/cluster.yaml b/pkg/config/testdata/test-cluster/cluster.yaml index 788ba588..c615369a 100644 --- a/pkg/config/testdata/test-cluster/cluster.yaml +++ b/pkg/config/testdata/test-cluster/cluster.yaml @@ -6,7 +6,6 @@ meta: Test cluster spec: - versionMajor: v0.10 bootstrapAddrs: - bootstrap-addr:9092 zkAddrs: From f76b65c67dcc15b1c3688d80d6b0f9ec1d537fe1 Mon Sep 17 00:00:00 2001 From: Benjamin Yolken Date: Wed, 25 Nov 2020 18:47:11 -0800 Subject: [PATCH 15/19] Update README --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 1814e126..7978ef37 100644 --- a/README.md +++ b/README.md @@ -238,8 +238,8 @@ spec: # ZooKeeper access settings (required for pre-v2 clusters) zkAddrs: # One or more cluster zookeeper addresses; if these are - - zk.example.com:2181 # omitted, then the cluster will only be accessed via broker APIs; - # see the section below on cluster access for more details. + - zk.example.com:2181 # omitted, then the cluster will only be accessed via broker APIs; + # see the section below on cluster access for more details. zkPrefix: my-cluster # Prefix for zookeeper nodes if using zookeeper access zkLockPath: /topicctl/locks # Path used for apply locks (optional) From 17a86603f668fef27b5b920031f128a3918fd9d4 Mon Sep 17 00:00:00 2001 From: Benjamin Yolken Date: Wed, 25 Nov 2020 19:06:57 -0800 Subject: [PATCH 16/19] Update README --- README.md | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 7978ef37..e077dff1 100644 --- a/README.md +++ b/README.md @@ -386,19 +386,19 @@ the process should continue from where it left off. ## Cluster access details -### zk vs. broker APIs +### ZooKeeper vs. broker APIs `topicctl` can interact with a cluster through either ZooKeeper or by hitting broker APIs directly. -Broker APIs are used exclusively if the tool is run with either: +Broker APIs are used exclusively if the tool is run with either of the following flags: 1. `--broker-addr` *or* 2. `--cluster-config` and the cluster config doesn't specify any ZK addresses In all other cases, i.e. if `--zk-addr` is specified or the cluster config has ZK addresses, then ZooKeeper will be used for most interactions. A few operations that are not possible via ZK -will still use broker APIs, including: +will still use broker APIs, however, including: 1. Group-related `get` commands: `get groups`, `get lags`, `get members` 2. `get offsets` @@ -406,8 +406,17 @@ will still use broker APIs, including: 4. `tail` 5. `apply` with topic creation -Note that the broker-only mode is only possible with newer Kafka versions (>= 2 for read-only -operations, >= 2.4 for applies). +### Limitations of broker-only access mode + +There are a few limitations in the tool when using the broker APIs exclusively: + +1. Only newer versions of Kafka are supported. In particular: + a. v2.0 or greater is required for read-only operations (`get brokers`, `get topics`, etc.) + b. v2.4 or greater is required for applying topic changes +2. Apply locking is not yet implemented; please be careful when applying to ensure that someone + else isn't applying changes in the same topic at the same time. +3. The values of some dynamic broker properties, e.g. `leader.replication.throttled.rate`, are not + returned by the API and thus won't appear in the tool output. This appears to be fixed in v2.6. ### TLS From bcaf12f3a6252fde80f931d49e03de20de5aa8a8 Mon Sep 17 00:00:00 2001 From: Benjamin Yolken Date: Wed, 25 Nov 2020 19:08:13 -0800 Subject: [PATCH 17/19] Update README --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e077dff1..d73e48ca 100644 --- a/README.md +++ b/README.md @@ -411,8 +411,8 @@ will still use broker APIs, however, including: There are a few limitations in the tool when using the broker APIs exclusively: 1. Only newer versions of Kafka are supported. In particular: - a. v2.0 or greater is required for read-only operations (`get brokers`, `get topics`, etc.) - b. v2.4 or greater is required for applying topic changes + a. v2.0 or greater is required for read-only operations (`get brokers`, `get topics`, etc.) + b. v2.4 or greater is required for applying topic changes 2. Apply locking is not yet implemented; please be careful when applying to ensure that someone else isn't applying changes in the same topic at the same time. 3. The values of some dynamic broker properties, e.g. `leader.replication.throttled.rate`, are not From 07701a409f0e2d158e7912f13534473672ce3a9f Mon Sep 17 00:00:00 2001 From: Benjamin Yolken Date: Wed, 25 Nov 2020 19:09:13 -0800 Subject: [PATCH 18/19] Update README --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d73e48ca..2cd26a3e 100644 --- a/README.md +++ b/README.md @@ -411,8 +411,8 @@ will still use broker APIs, however, including: There are a few limitations in the tool when using the broker APIs exclusively: 1. Only newer versions of Kafka are supported. In particular: - a. v2.0 or greater is required for read-only operations (`get brokers`, `get topics`, etc.) - b. v2.4 or greater is required for applying topic changes + 1. v2.0 or greater is required for read-only operations (`get brokers`, `get topics`, etc.) + 2. v2.4 or greater is required for applying topic changes 2. Apply locking is not yet implemented; please be careful when applying to ensure that someone else isn't applying changes in the same topic at the same time. 3. The values of some dynamic broker properties, e.g. `leader.replication.throttled.rate`, are not From dfdb2cf1ffb207e158e8488a8a6362d9a2413091 Mon Sep 17 00:00:00 2001 From: Benjamin Yolken Date: Wed, 25 Nov 2020 19:10:14 -0800 Subject: [PATCH 19/19] Update README --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 2cd26a3e..a9006dba 100644 --- a/README.md +++ b/README.md @@ -411,8 +411,8 @@ will still use broker APIs, however, including: There are a few limitations in the tool when using the broker APIs exclusively: 1. Only newer versions of Kafka are supported. In particular: - 1. v2.0 or greater is required for read-only operations (`get brokers`, `get topics`, etc.) - 2. v2.4 or greater is required for applying topic changes + - v2.0 or greater is required for read-only operations (`get brokers`, `get topics`, etc.) + - v2.4 or greater is required for applying topic changes 2. Apply locking is not yet implemented; please be careful when applying to ensure that someone else isn't applying changes in the same topic at the same time. 3. The values of some dynamic broker properties, e.g. `leader.replication.throttled.rate`, are not