Skip to content

Commit 77603f9

Browse files
committed
Some updates
1 parent 003d20a commit 77603f9

File tree

8 files changed

+103
-51
lines changed

8 files changed

+103
-51
lines changed

Makefile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ $(CMD_DIR): FORCE
3939

4040
$(PLUGIN_DIR): FORCE
4141
@echo Build plugin $(notdir $@)
42-
${GO} build -buildmode=plugin -o ${BUILD_DIR}/$(notdir $@).plugin ${BUILD_FLAGS} ./$@
42+
@${GO} build -buildmode=plugin -o ${BUILD_DIR}/$(notdir $@).plugin ${BUILD_FLAGS} ./$@
4343

4444
FORCE:
4545

etc/server.yaml

Lines changed: 4 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -30,8 +30,7 @@ handlers:
3030
sqlite3:
3131
# Databases to load and/or create. Only the 'main' database is required.
3232
databases:
33-
main: /tmp/test.sqlite
34-
test: ":memory:"
33+
main: "/tmp/test.sqlite"
3534

3635
# Set create to true to allow databases which don't exist to be created, or
3736
# else error will be reported on server start. In-memory databases can always
@@ -40,16 +39,12 @@ sqlite3:
4039

4140
# Set trace to true to enable the ability to profile queries. Profiling information
4241
# can be displayed through the API.
43-
trace: false
42+
trace: true
4443

4544
# Set max number of connections that can be simultaneously opened
4645
max: 100
4746

4847
indexer:
48+
workers: 2
4949
index:
50-
sw: /home/djt/sw
51-
tv: /home/djt/media/TV
52-
films: /home/djt/media/Movies
53-
music: /home/djt/media/Music
54-
videos: /home/djt/media/Videos
55-
media: /home/djt/media
50+
home: /Volumes/Scratch/Downloads

pkg/indexer/schema.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,7 @@ func CreateSchema(ctx context.Context, conn SQConnection, schema string, tokeniz
8080
"name",
8181
"parent",
8282
"filename",
83+
).Options(
8384
"content="+filesTableName,
8485
"tokenize="+Quote(tokenizer),
8586
).IfNotExists()); err != nil {

pkg/lang/indexview.go

Lines changed: 20 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ type createvirtual struct {
2525
module string
2626
ifnotexists bool
2727
args []string
28+
opts []string
2829
}
2930

3031
///////////////////////////////////////////////////////////////////////////////
@@ -37,7 +38,7 @@ func (this *source) CreateIndex(name string, columns ...string) SQIndexView {
3738

3839
// Create a virtual table with module name name and arguments
3940
func (this *source) CreateVirtualTable(module string, args ...string) SQIndexView {
40-
return &createvirtual{source{this.name, this.schema, "", false}, module, false, args}
41+
return &createvirtual{source{this.name, this.schema, "", false}, module, false, args, nil}
4142
}
4243

4344
////////////////////////////////////////////////////////////////////////////////
@@ -99,22 +100,30 @@ func (this *createindex) WithAuto() SQIndexView {
99100
return &createindex{this.source, this.name, true, this.ifnotexists, this.columns, true}
100101
}
101102

103+
func (this *createindex) Options(opts ...string) SQIndexView {
104+
return nil
105+
}
106+
102107
func (this *createvirtual) IfNotExists() SQIndexView {
103-
return &createvirtual{this.source, this.module, true, this.args}
108+
return &createvirtual{this.source, this.module, true, this.args, this.opts}
104109
}
105110

106111
func (this *createvirtual) WithUnique() SQIndexView {
107112
return nil
108113
}
109114

110115
func (this *createvirtual) WithTemporary() SQIndexView {
111-
return &createvirtual{source{this.name, "temp", "", false}, this.module, this.ifnotexists, this.args}
116+
return &createvirtual{source{this.name, "temp", "", false}, this.module, this.ifnotexists, this.args, this.opts}
112117
}
113118

114119
func (this *createvirtual) WithAuto() SQIndexView {
115120
return nil
116121
}
117122

123+
func (this *createvirtual) Options(opts ...string) SQIndexView {
124+
return &createvirtual{source{this.name, this.schema, "", false}, this.module, this.ifnotexists, this.args, opts}
125+
}
126+
118127
////////////////////////////////////////////////////////////////////////////////
119128
// STRINGIFY
120129

@@ -153,8 +162,15 @@ func (this *createvirtual) Query() string {
153162
tokens = append(tokens, "IF NOT EXISTS")
154163
}
155164
tokens = append(tokens, this.source.String(), "USING", QuoteIdentifier(this.module))
165+
argsopts := []string{}
156166
if len(this.args) > 0 {
157-
tokens = append(tokens, "("+QuoteIdentifiers(this.args...)+")")
167+
argsopts = append(argsopts, QuoteIdentifiers(this.args...))
168+
}
169+
if len(this.opts) > 0 {
170+
argsopts = append(argsopts, strings.Join(this.opts, ","))
171+
}
172+
if len(argsopts) > 0 {
173+
tokens = append(tokens, "(", strings.Join(argsopts, ",")+")")
158174
}
159175

160176
// Return the query

pkg/sqlite3/conn.go

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,12 @@ func OpenPath(path string, flags SQFlag) (*Conn, error) {
6666
}
6767
}
6868

69+
// If we are opening a memory database, then we need to set the flags
70+
if path == defaultMemory {
71+
path = "file:" + DefaultSchema
72+
flags |= SQFlag(sqlite3.SQLITE_OPEN_MEMORY | sqlite3.SQLITE_OPEN_URI)
73+
}
74+
6975
// Open database with flags
7076
if c, err := sqlite3.OpenPathEx(path, sqlite3.OpenFlags(flags), ""); err != nil {
7177
return nil, err

plugin/indexer/main.go

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,9 @@ import (
2222
// TYPES
2323

2424
type Config struct {
25-
Paths map[string]string `yaml:"index"`
26-
Schema string `yaml:"database"`
25+
Workers uint `json:"workers"`
26+
Paths map[string]string `yaml:"index"`
27+
Schema string `yaml:"database"`
2728
}
2829

2930
type plugin struct {
@@ -88,7 +89,7 @@ func New(ctx context.Context, provider Provider) Plugin {
8889
if q == nil {
8990
provider.Print(ctx, "unable to create queue")
9091
return nil
91-
} else if store := indexer.NewStore(p.pool, schema, q, nil, 0); store == nil {
92+
} else if store := indexer.NewStore(p.pool, schema, q, nil, cfg.Workers); store == nil {
9293
provider.Print(ctx, "unable to create store")
9394
return nil
9495
} else {

sqlang.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,7 @@ type SQIndexView interface {
103103
WithTemporary() SQIndexView
104104
WithUnique() SQIndexView
105105
WithAuto() SQIndexView
106+
Options(...string) SQIndexView
106107
}
107108

108109
// SQTrigger defines a create trigger statement

sys/sqlite3/statement.go

Lines changed: 66 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -19,66 +19,94 @@ import (
1919
// sqlite library needs to be compiled with -DSQLITE_ENABLE_UNLOCK_NOTIFY
2020
// https://www.sqlite.org/unlock_notify.html
2121
22-
typedef struct UnlockNotification UnlockNotification;
22+
// A pointer to an instance of this structure is passed as the user-context
23+
// pointer when registering for an unlock-notify callback.
2324
25+
typedef struct UnlockNotification UnlockNotification;
2426
struct UnlockNotification {
25-
int fired; // True after unlock event has occurred
26-
pthread_cond_t cond; // Condition variable to wait on
27-
pthread_mutex_t mutex; // Mutex to protect structure
27+
int fired; // True after unlock event has occurred
28+
pthread_cond_t cond; // Condition variable to wait on
29+
pthread_mutex_t mutex; // Mutex to protect structure
2830
};
2931
30-
static void _unlock_notify_cb(void** apArg, int nArg) {
31-
int i;
32-
for(i = 0; i < nArg; i++) {
33-
UnlockNotification *p = (UnlockNotification* )apArg[i];
34-
pthread_mutex_lock(&p->mutex);
35-
p->fired = 1;
36-
pthread_cond_signal(&p->cond);
37-
pthread_mutex_unlock(&p->mutex);
32+
// This function is an unlock-notify callback registered with SQLite.
33+
static void unlock_notify_cb(void **apArg, int nArg){
34+
for(int i = 0; i < nArg; i++){
35+
UnlockNotification *p = (UnlockNotification *)apArg[i];
36+
pthread_mutex_lock(&p->mutex);
37+
p->fired = 1;
38+
pthread_cond_signal(&p->cond);
39+
pthread_mutex_unlock(&p->mutex);
3840
}
3941
}
4042
41-
int _wait_for_unlock_notify(sqlite3* db) {
43+
// This function assumes that an SQLite API call (either sqlite3_prepare_v2()
44+
// or sqlite3_step()) has just returned SQLITE_LOCKED. The argument is the
45+
// associated database connection.
46+
//
47+
// This function calls sqlite3_unlock_notify() to register for an
48+
// unlock-notify callback, then blocks until that callback is delivered
49+
// and returns SQLITE_OK. The caller should then retry the failed operation.
50+
//
51+
// Or, if sqlite3_unlock_notify() indicates that to block would deadlock
52+
// the system, then this function returns SQLITE_LOCKED immediately. In
53+
// this case the caller should not retry the operation and should roll
54+
// back the current transaction (if any).
55+
static int wait_for_unlock_notify(sqlite3 *db) {
4256
int rc;
4357
UnlockNotification un;
4458
45-
// Initialize the UnlockNotification structure
59+
// Initialize the UnlockNotification structure.
4660
un.fired = 0;
4761
pthread_mutex_init(&un.mutex, 0);
4862
pthread_cond_init(&un.cond, 0);
4963
50-
// Register for an unlock-notify callback
64+
// Register for an unlock-notify callback.
65+
rc = sqlite3_unlock_notify(db, unlock_notify_cb, (void *)&un);
66+
assert( rc==SQLITE_LOCKED || rc==SQLITE_OK );
67+
5168
// The call to sqlite3_unlock_notify() always returns either SQLITE_LOCKED
52-
// or SQLITE_OK
53-
rc = sqlite3_unlock_notify(db, _unlock_notify_cb, (void* )&un);
54-
assert(rc == SQLITE_LOCKED || rc == SQLITE_OK);
55-
if(rc == SQLITE_OK) {
56-
pthread_mutex_lock(&un.mutex);
57-
if (!un.fired) {
69+
// or SQLITE_OK.
70+
//
71+
// If SQLITE_LOCKED was returned, then the system is deadlocked. In this
72+
// case this function needs to return SQLITE_LOCKED to the caller so
73+
// that the current transaction can be rolled back. Otherwise, block
74+
// until the unlock-notify callback is invoked, then return SQLITE_OK.
75+
if( rc==SQLITE_OK ){
76+
pthread_mutex_lock(&un.mutex);
77+
if( !un.fired ){
5878
pthread_cond_wait(&un.cond, &un.mutex);
5979
}
6080
pthread_mutex_unlock(&un.mutex);
6181
}
6282
63-
// Destroy the mutex and condition variables
83+
// Destroy the mutex and condition variables.
6484
pthread_cond_destroy(&un.cond);
6585
pthread_mutex_destroy(&un.mutex);
86+
6687
return rc;
6788
}
6889
90+
6991
// This code is a wrapper around sqlite3_step
7092
static int _sqlite3_blocking_step(sqlite3_stmt* stmt) {
71-
while (1) {
72-
int rc = sqlite3_step(stmt);
73-
if ((rc & 0xFF) != SQLITE_LOCKED) {
74-
return rc;
93+
int rc;
94+
sqlite3* db = sqlite3_db_handle(stmt);
95+
for (;;) {
96+
rc = sqlite3_step(stmt);
97+
if (rc != SQLITE_LOCKED) {
98+
break;
7599
}
76-
rc = _wait_for_unlock_notify(sqlite3_db_handle(stmt));
100+
if (sqlite3_extended_errcode(db) != SQLITE_LOCKED_SHAREDCACHE) {
101+
break;
102+
}
103+
rc = wait_for_unlock_notify(db);
77104
if (rc != SQLITE_OK) {
78-
return rc;
105+
break;
79106
}
80107
sqlite3_reset(stmt);
81108
}
109+
return rc;
82110
}
83111
84112
// This code is a wrapper around sqlite3_prepare_v2
@@ -90,16 +118,20 @@ static int _sqlite3_blocking_prepare_v2(
90118
const char** pz // OUT: End of parsed string
91119
){
92120
int rc;
93-
while (1) {
94-
int rc = sqlite3_prepare_v2(db, sql, nSql, stmt, pz);
95-
if ((rc & 0xFF) != SQLITE_LOCKED) {
96-
return rc;
121+
for (;;) {
122+
rc = sqlite3_prepare_v2(db, sql, nSql, stmt, pz);
123+
if (rc != SQLITE_LOCKED) {
124+
break;
97125
}
98-
rc = _wait_for_unlock_notify(db);
126+
if (sqlite3_extended_errcode(db) != SQLITE_LOCKED_SHAREDCACHE) {
127+
break;
128+
}
129+
rc = wait_for_unlock_notify(db);
99130
if (rc != SQLITE_OK) {
100-
return rc;
131+
break;
101132
}
102133
}
134+
return rc;
103135
}
104136
*/
105137
import "C"

0 commit comments

Comments
 (0)