-
Notifications
You must be signed in to change notification settings - Fork 499
/
read.go
63 lines (52 loc) · 1.29 KB
/
read.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
package client
import (
"context"
"fmt"
"github.com/apache/arrow/go/v16/arrow"
"github.com/apache/arrow/go/v16/arrow/array"
"github.com/apache/arrow/go/v16/arrow/memory"
"github.com/cloudquery/plugin-sdk/v4/schema"
"github.com/meilisearch/meilisearch-go"
)
func (c *Client) Read(_ context.Context, table *schema.Table, res chan<- arrow.Record) error {
sc := table.ToArrowSchema()
index, err := c.Meilisearch.GetIndex(table.Name)
if err != nil {
return err
}
req := &meilisearch.SearchRequest{
HitsPerPage: 100, // default = 1, we want more
Page: 1, // starting from 1
}
for {
resp, err := index.Search("", req)
if err != nil {
return err
}
for _, hit := range resp.Hits {
m, ok := hit.(map[string]any)
if !ok {
return fmt.Errorf("unsupported format for doc: %T", hit)
}
row, err := docToRecord(sc, m)
if err != nil {
return err
}
res <- row
}
if resp.TotalPages == req.Page {
break
}
req.Page++
}
return nil
}
func docToRecord(sc *arrow.Schema, doc map[string]any) (arrow.Record, error) {
builder := array.NewRecordBuilder(memory.DefaultAllocator, sc)
for i, builder := range builder.Fields() {
if err := reverseTransform(builder, doc[sc.Field(i).Name]); err != nil {
return nil, err
}
}
return builder.NewRecord(), nil
}