Skip to content

Commit 4ab7c4b

Browse files
djthorpedjthorpe
committed
Updated indexer code
1 parent aea634b commit 4ab7c4b

File tree

5 files changed

+202
-4
lines changed

5 files changed

+202
-4
lines changed

‎etc/server.yaml‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,13 @@ handlers:
2020
middleware:
2121
- log
2222

23+
indexer:
24+
# The indexer plugin can be viewed at http://localhost/api/indexer
25+
prefix: /api/indexer
26+
# Requests are logged
27+
middleware:
28+
- log
29+
2330
sqlite3:
2431
# Databases to load and/or create. Only the 'main' database is required.
2532
databases:
@@ -33,12 +40,17 @@ sqlite3:
3340

3441
# Set trace to true to enable the ability to profile queries. Profiling information
3542
# can be displayed through the API.
36-
trace: true
43+
trace: false
3744

3845
# Set max number of connections that can be simultaneously opened
39-
max: 10
46+
max: 100
4047

4148

4249
indexer:
4350
index:
51+
sw: /home/djt/sw
52+
tv: /home/djt/media/TV
53+
films: /home/djt/media/Movies
54+
music: /home/djt/media/Music
55+
videos: /home/djt/media/Videos
4456
media: /home/djt/media

‎pkg/indexer/schema.go‎

Lines changed: 55 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package indexer
33
import (
44
"context"
55
"errors"
6+
"io"
67
"path/filepath"
78

89
// Namespace imports
@@ -14,7 +15,11 @@ import (
1415
// GLOBALS
1516

1617
const (
17-
filesTableName = "files"
18+
filesTableName = "file"
19+
nameIndexName = "file_name"
20+
parentIndexName = "file_parent"
21+
filenameIndexName = "file_filename"
22+
extIndexName = "file_filename"
1823
)
1924

2025
///////////////////////////////////////////////////////////////////////////////
@@ -27,7 +32,7 @@ func CreateSchema(ctx context.Context, pool SQPool, schema string) error {
2732
}
2833
defer pool.Put(conn)
2934

30-
// Create table
35+
// Create tables
3136
return conn.Do(ctx, 0, func(txn SQTransaction) error {
3237
if _, err := txn.Query(N(filesTableName).WithSchema(schema).CreateTable(
3338
C("name").WithPrimary(),
@@ -41,10 +46,58 @@ func CreateSchema(ctx context.Context, pool SQPool, schema string) error {
4146
).IfNotExists()); err != nil {
4247
return err
4348
}
49+
// Create the indexes
50+
if _, err := txn.Query(N(nameIndexName).WithSchema(schema).CreateIndex(
51+
filesTableName, "name",
52+
).IfNotExists()); err != nil {
53+
return err
54+
}
55+
if _, err := txn.Query(N(parentIndexName).WithSchema(schema).CreateIndex(
56+
filesTableName, "parent",
57+
).IfNotExists()); err != nil {
58+
return err
59+
}
60+
if _, err := txn.Query(N(filenameIndexName).WithSchema(schema).CreateIndex(
61+
filesTableName, "filename",
62+
).IfNotExists()); err != nil {
63+
return err
64+
}
65+
if _, err := txn.Query(N(extIndexName).WithSchema(schema).CreateIndex(
66+
filesTableName, "ext",
67+
).IfNotExists()); err != nil {
68+
return err
69+
}
4470
return nil
4571
})
4672
}
4773

74+
// Get indexes and count of documents for each index
75+
func ListIndexWithCount(ctx context.Context, conn SQConnection, schema string) (map[string]int64, error) {
76+
results := make(map[string]int64)
77+
if err := conn.Do(ctx, 0, func(txn SQTransaction) error {
78+
s := Q("SELECT name,COUNT(*) AS count FROM ", N(filesTableName).WithSchema(schema), " GROUP BY name")
79+
r, err := txn.Query(s)
80+
if err != nil && err != io.EOF {
81+
return err
82+
}
83+
for {
84+
row, err := r.Next()
85+
if err != nil {
86+
break
87+
}
88+
if len(row) == 2 {
89+
results[row[0].(string)] = row[1].(int64)
90+
}
91+
}
92+
return nil
93+
}); err != nil {
94+
return nil, err
95+
}
96+
97+
// Return success
98+
return results, nil
99+
}
100+
48101
func Replace(schema string, evt *QueueEvent) (SQStatement, []interface{}) {
49102
return N(filesTableName).WithSchema(schema).Replace("name", "path", "parent", "filename", "isdir", "ext", "modtime", "size"),
50103
[]interface{}{

‎pkg/indexer/store.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,13 @@ func (s *Store) Run(ctx context.Context, errs chan<- error) error {
105105
return result
106106
}
107107

108+
///////////////////////////////////////////////////////////////////////////////
109+
// PUBLIC METHODS
110+
111+
func (s *Store) Schema() string {
112+
return s.schema
113+
}
114+
108115
///////////////////////////////////////////////////////////////////////////////
109116
// PRIVATE METHODS
110117

‎plugin/indexer/handlers.go‎

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
package main
2+
3+
import (
4+
"context"
5+
"net/http"
6+
"regexp"
7+
8+
// Packages
9+
router "github.com/mutablelogic/go-server/pkg/httprouter"
10+
indexer "github.com/mutablelogic/go-sqlite/pkg/indexer"
11+
12+
// Namespace imports
13+
. "github.com/mutablelogic/go-server"
14+
)
15+
16+
///////////////////////////////////////////////////////////////////////////////
17+
// TYPES
18+
19+
type PingResponse struct {
20+
Indexes []IndexResponse `json:"indexes"`
21+
}
22+
23+
type IndexResponse struct {
24+
Name string `json:"name"`
25+
Path string `json:"path,omitempty"`
26+
Count int64 `json:"count,omitempty"`
27+
Modtime interface{} `json:"reindexed,omitempty"`
28+
Status string `json:"status,omitempty"`
29+
}
30+
31+
///////////////////////////////////////////////////////////////////////////////
32+
// ROUTES
33+
34+
var (
35+
reRoutePing = regexp.MustCompile(`^/?$`)
36+
)
37+
38+
///////////////////////////////////////////////////////////////////////////////
39+
// CONSTANTS
40+
41+
const (
42+
maxResultLimit = 1000
43+
)
44+
45+
///////////////////////////////////////////////////////////////////////////////
46+
// LIFECYCLE
47+
48+
func (p *plugin) AddHandlers(ctx context.Context, provider Provider) error {
49+
// Add handler for ping
50+
if err := provider.AddHandlerFuncEx(ctx, reRoutePing, p.ServePing); err != nil {
51+
return err
52+
}
53+
54+
// Return success
55+
return nil
56+
}
57+
58+
///////////////////////////////////////////////////////////////////////////////
59+
// HANDLERS
60+
61+
func (p *plugin) ServePing(w http.ResponseWriter, req *http.Request) {
62+
// Get a connection
63+
conn := p.pool.Get()
64+
if conn == nil {
65+
router.ServeError(w, http.StatusBadGateway, "No connection")
66+
return
67+
}
68+
defer p.pool.Put(conn)
69+
70+
// Retrieve indexes with count of documents in each
71+
index, err := indexer.ListIndexWithCount(req.Context(), conn, p.store.Schema())
72+
if err != nil {
73+
router.ServeError(w, http.StatusBadGateway, err.Error())
74+
return
75+
}
76+
77+
// Add known indexes to the response - these may not yet have any rows in the
78+
// database
79+
for _, idx := range p.index {
80+
name := idx.Name()
81+
if _, exists := index[name]; !exists {
82+
index[name] = 0
83+
}
84+
}
85+
86+
// Populate response
87+
response := PingResponse{
88+
Indexes: make([]IndexResponse, 0, len(index)),
89+
}
90+
for name, count := range index {
91+
response.Indexes = append(response.Indexes, IndexResponse{
92+
Name: name,
93+
Count: count,
94+
Path: p.pathForIndex(name),
95+
Modtime: p.modtimeForIndex(name),
96+
})
97+
}
98+
// Now add all indexes into the response, adding their modtime
99+
100+
// Serve response
101+
router.ServeJSON(w, response, http.StatusOK, 2)
102+
}
103+
104+
///////////////////////////////////////////////////////////////////////////////
105+
// PRIVATE METHODS
106+
107+
func (p *plugin) pathForIndex(name string) string {
108+
if idx, exists := p.index[name]; exists {
109+
return idx.Path()
110+
} else {
111+
return ""
112+
}
113+
}
114+
115+
func (p *plugin) modtimeForIndex(name string) interface{} {
116+
if t, exists := p.modtime[name]; exists && t.IsZero() == false {
117+
return t
118+
} else {
119+
return nil
120+
}
121+
}

‎plugin/indexer/main.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,11 @@ func (p *plugin) Run(ctx context.Context, provider Provider) error {
137137
}
138138
}()
139139

140+
// Add handlers
141+
if err := p.AddHandlers(ctx, provider); err != nil {
142+
return err
143+
}
144+
140145
// Run indexer processes
141146
for _, idx := range p.index {
142147
wg.Add(1)

0 commit comments

Comments
 (0)