db, refactor: remove fulltext search in sqlite.

fibjs · Jan 18, 2023 · 1596f13 · 1596f13
1 parent cc15203
commit 1596f13
Show file tree

Hide file tree

Showing 3 changed files with 2 additions and 97 deletions.
diff --git a/fibjs/src/db/sql/SQLite.cpp b/fibjs/src/db/sql/SQLite.cpp
@@ -14,80 +14,7 @@
 
 namespace fibjs {
 
-#define TOKEN(s, n) pCtx1->xToken(pCtx1->pCtx, tflags, s, (int)(n - s), (int)(s - pToken) + iStart, (int)(n - pToken) + iStart)
-
-struct MyCtx {
-    void* pCtx;
-    int (*xToken)(void*, int, const char*, int, int, int);
-};
-
-static int new_Token_cb(void* pCtx, int tflags, const char* pToken, int nToken, int iStart, int iEnd)
-{
-    MyCtx* pCtx1 = (MyCtx*)pCtx;
-
-    const char* pStart = pToken;
-    const char* pEnd = pToken + nToken;
-    const char* pAscii = pStart;
-
-    while (pStart < pEnd) {
-        const char* p = pStart;
-        exlib::wchar32 ch = utf_getchar(pStart, pEnd);
-
-        if (ch >= 0x4e00 && ch <= 0xa000) {
-            if (p > pAscii)
-                TOKEN(pAscii, p);
-            TOKEN(p, pStart);
-
-            pAscii = pStart;
-        }
-    }
-    if (pEnd > pAscii)
-        TOKEN(pAscii, pEnd);
-
-    return 0;
-}
-
-static int (*old_tokenize)(Fts5Tokenizer*, void* pCtx, int flags, const char* pText, int nText,
-    int (*xToken)(void*, int, const char*, int, int, int));
-static int new_tokenize(Fts5Tokenizer* pTokenizer, void* pCtx, int iUnused, const char* pText, int nText,
-    int (*xToken)(void*, int, const char*, int nToken, int iStart, int iEnd))
-{
-    MyCtx ctx = { pCtx, xToken };
-    return old_tokenize(pTokenizer, &ctx, iUnused, pText, nText, new_Token_cb);
-}
-
-static int (*old_fts5FindTokenizer)(fts5_api* pApi, const char* zName, void** ppUserData, fts5_tokenizer* pTokenizer);
-static int new_fts5FindTokenizer(fts5_api* pApi, const char* zName, void** ppUserData, fts5_tokenizer* pTokenizer)
-{
-    int rc = old_fts5FindTokenizer(pApi, zName, ppUserData, pTokenizer);
-    if (rc == 0 && !qstrcmp(zName, "unicode61")) {
-        if (old_tokenize == NULL)
-            old_tokenize = pTokenizer->xTokenize;
-        pTokenizer->xTokenize = new_tokenize;
-    }
-
-    return rc;
-}
-
-static void hook_fts5_api(sqlite3* db)
-{
-    fts5_api* pApi = 0;
-    sqlite3_stmt* pStmt = 0;
-
-    if (SQLITE_OK == sqlite3_prepare(db, "SELECT fts5(?)", -1, &pStmt, 0)) {
-        sqlite3_bind_pointer(pStmt, 1, (void*)&pApi, "fts5_api_ptr", NULL);
-        sqlite3_step(pStmt);
-
-        if (old_fts5FindTokenizer == NULL)
-            old_fts5FindTokenizer = pApi->xFindTokenizer;
-        pApi->xFindTokenizer = new_fts5FindTokenizer;
-
-        sqlite3_finalize(pStmt);
-    }
-}
-
-#define SQLITE_OPEN_FLAGS \
-    SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_SHAREDCACHE | SQLITE_OPEN_NOMUTEX
+#define SQLITE_OPEN_FLAGS SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_SHAREDCACHE | SQLITE_OPEN_NOMUTEX
 
 result_t db_base::openSQLite(exlib::string connString,
     obj_ptr<SQLite_base>& retVal, AsyncEvent* ac)
@@ -125,8 +52,6 @@ result_t SQLite::open(const char* file)
     obj_ptr<NArray> retVal;
     cc_execute("PRAGMA journal_mode=WAL;", retVal);
 
-    hook_fts5_api((sqlite3*)m_conn);
-
     m_file = file;
 
     return 0;

diff --git a/test/db_test.js b/test/db_test.js
@@ -1356,26 +1356,6 @@ describe("db", () => {
             assert.equal(journal_mode, "wal");
         });
 
-        it("fulltext search", () => {
-            var conn = db.open(conn_str);
-
-            conn.execute('CREATE VIRTUAL TABLE email USING fts5(sender, title, body)');
-            conn.execute('insert into email(sender, title, body) values("tom cat", "some titles", "this is body")');
-            conn.execute('insert into email(sender, title, body) values("响马", "这里是标题yes", "this这里是身体")');
-            assert.deepEqual(conn.execute('SELECT * FROM email WHERE email MATCH "这里"'), []);
-
-            conn.execute('CREATE VIRTUAL TABLE email1 USING fts5(sender, title, body, tokenize = porter)');
-            conn.execute('insert into email1(sender, title, body) values("tom cat", "some titles", "this is body")');
-            conn.execute('insert into email1(sender, title, body) values("响马", "这里是标题yes", "this这里是身体")');
-            assert.deepEqual(conn.execute('SELECT * FROM email1 WHERE email1 MATCH "这里"'), [{
-                "body": "this这里是身体",
-                "sender": "响马",
-                "title": "这里是标题yes"
-            }]);
-
-            conn.close();
-        });
-
         it("backup", () => {
             var conn = db.open(conn_str);
             conn.backup(conn_str + ".backup");

diff --git a/vender b/vender