-
Notifications
You must be signed in to change notification settings - Fork 500
/
read.go
42 lines (38 loc) · 905 Bytes
/
read.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
package client
import (
"bytes"
"context"
"time"
"github.com/IBM/sarama"
"github.com/apache/arrow/go/v16/arrow"
"github.com/cloudquery/plugin-sdk/v4/schema"
)
const (
maxWaitTime = 3 * time.Second
)
func (c *Client) Read(ctx context.Context, table *schema.Table, res chan<- arrow.Record) error {
consumer, err := sarama.NewConsumer(c.spec.Brokers, c.conf)
if err != nil {
return err
}
defer consumer.Close()
partitionConsumer, err := consumer.ConsumePartition(table.Name, 0, sarama.OffsetOldest)
if err != nil {
return err
}
defer partitionConsumer.Close()
for {
select {
case <-ctx.Done():
return ctx.Err()
case msg := <-partitionConsumer.Messages():
if err := c.Client.Read(bytes.NewReader(msg.Value), table, res); err != nil {
return err
}
case err := <-partitionConsumer.Errors():
return err.Err
case <-time.After(maxWaitTime):
return nil
}
}
}