Skip to content

Commit

Permalink
works for node, not for browser
Browse files Browse the repository at this point in the history
  • Loading branch information
fergiemcdowall committed Aug 8, 2023
1 parent 0efb00a commit 1bb22a4
Show file tree
Hide file tree
Showing 41 changed files with 555 additions and 299 deletions.
22 changes: 22 additions & 0 deletions package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

11 changes: 7 additions & 4 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,9 @@
"level-read-stream": "1.1.0",
"traverse": "0.6.7"
},
"directories": {
"lib": "src"
},
"files": [
"src"
],
Expand All @@ -20,6 +23,7 @@
"level-out": "^1.0.1",
"os-browserify": "^0.3.0",
"path-browserify": "^1.0.1",
"prettier": "^2.8.8",
"process": "^0.11.10",
"standard": "17.1.0",
"stemmer": "2.0.1",
Expand All @@ -31,6 +35,7 @@
"webpack-cli": "^5.1.4",
"world-bank-dataset": "^1.0.0"
},
"prettier": "./.prettierrc.json",
"scripts": {
"build": "npm run empty-sandbox && webpack",
"empty-sandbox": "rm -rf test/sandbox && mkdir test/sandbox",
Expand All @@ -40,6 +45,7 @@
"test-node": "npm run empty-sandbox && tape test/src/*.js",
"test-script-tag-lib": "npm run build && open-cli test/src/index.html"
},
"type": "module",
"repository": {
"type": "git",
"url": "git+https://github.com/fergiemcdowall/fergies-inverted-index"
Expand All @@ -49,8 +55,5 @@
"bugs": {
"url": "https://github.com/fergiemcdowall/fergies-inverted-index/issues"
},
"homepage": "https://github.com/fergiemcdowall/fergies-inverted-index",
"resolutions": {
"minimist": "^1.2.8"
}
"homepage": "https://github.com/fergiemcdowall/fergies-inverted-index"
}
17 changes: 11 additions & 6 deletions src/browser.js
Original file line number Diff line number Diff line change
@@ -1,7 +1,12 @@
const fii = require('./main.js')
const { BrowserLevel } = require('browser-level')
// import levelOptions from './options.js'
import { Main as MainInvertedIndex } from './main.js'
import { BrowserLevel } from 'browser-level'

module.exports = ({ name = 'fii', ...ops }) => fii({
db: new BrowserLevel(name),
...ops
})
export class InvertedIndex {
constructor(ops = {}) {
return new MainInvertedIndex({
db: new BrowserLevel(ops.name || 'fii', { valueEncoding: 'json' }),
...ops
})
}
}
48 changes: 27 additions & 21 deletions src/main.js
Original file line number Diff line number Diff line change
@@ -1,29 +1,31 @@
// const level = require('level')
const read = require('./read.js')
const write = require('./write.js')
const levelOptions = require('./options.js')
// const read = require('./read.js')
// const write = require('./write.js')
// const levelOptions = require('./options.js')

import read from './read.js'
import write from './write.js'

// _match is nested by default so that AND and OR work correctly under
// the bonnet. Flatten array before presenting to consumer
const flattenMatchArrayInResults = results =>
typeof results === 'undefined'
? undefined
: results.map(result => {
// Sort _match consistently (FIELD -> VALUE -> SCORE)
result._match = result._match
.flat(Infinity)
.map(m => (typeof m === 'string' ? JSON.parse(m) : m))
.sort((a, b) => {
if (a.FIELD < b.FIELD) return -1
if (a.FIELD > b.FIELD) return 1
if (a.VALUE < b.VALUE) return -1
if (a.VALUE > b.VALUE) return 1
if (a.SCORE < b.SCORE) return -1
if (a.SCORE > b.SCORE) return 1
return 0
})
return result
})
// Sort _match consistently (FIELD -> VALUE -> SCORE)
result._match = result._match
.flat(Infinity)
.map(m => (typeof m === 'string' ? JSON.parse(m) : m))
.sort((a, b) => {
if (a.FIELD < b.FIELD) return -1
if (a.FIELD > b.FIELD) return 1
if (a.VALUE < b.VALUE) return -1
if (a.VALUE > b.VALUE) return 1
if (a.SCORE < b.SCORE) return -1
if (a.SCORE > b.SCORE) return 1
return 0
})
return result
})

const initStore = (ops = {}) =>
new Promise((resolve, reject) => {
Expand Down Expand Up @@ -72,7 +74,7 @@ const makeAFii = ops => {
r.GET(tokens, pipeline).then(flattenMatchArrayInResults),
IMPORT: w.IMPORT,
LAST_UPDATED: r.LAST_UPDATED,
LEVEL_OPTIONS: levelOptions,
// LEVEL_OPTIONS: levelOptions,
MAX: r.MAX,
MIN: r.MIN,
NOT: (...keys) =>
Expand All @@ -91,4 +93,8 @@ const makeAFii = ops => {
}))
}

module.exports = ops => initStore(ops).then(makeAFii)
export class Main {
constructor(ops) {
return initStore(ops).then(makeAFii)
}
}
17 changes: 11 additions & 6 deletions src/node.js
Original file line number Diff line number Diff line change
@@ -1,7 +1,12 @@
const fii = require('./main.js')
const { MemoryLevel } = require('memory-level')
import levelOptions from './options.js'
import { Main } from './main.js'
import { ClassicLevel } from 'classic-level'

module.exports = ({ name = 'fii', ...ops }) => fii({
db: new MemoryLevel(),
...ops
})
export class InvertedIndex {
constructor(ops = {}) {
return new Main({
db: new ClassicLevel(ops.name || 'fii', levelOptions),
...ops
})
}
}
4 changes: 2 additions & 2 deletions src/options.js
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
const charwise = require('charwise')
import charwise from 'charwise'

module.exports = {
export default {
keyEncoding: charwise,
valueEncoding: 'json'
}
6 changes: 4 additions & 2 deletions src/parseToken.js
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,9 @@ charwise.HI = undefined
// key might be object or string like this
// <fieldname>:<value>. Turn key into json object that is of the
// format {FIELD: ..., VALUE: {GTE: ..., LTE ...}}
module.exports = (token, availableFields) =>
new Promise((resolve, reject) => {
// module.exports = (token, availableFields) =>
export default function (token, availableFields) {
return new Promise((resolve, reject) => {
// case: <value>
// case: <FIELD>:<VALUE>
// case: undefined
Expand Down Expand Up @@ -106,3 +107,4 @@ module.exports = (token, availableFields) =>

return resolve(token)
})
}
28 changes: 19 additions & 9 deletions src/read.js
Original file line number Diff line number Diff line change
@@ -1,13 +1,20 @@
const tokenParser = require('./parseToken.js')
const charwise = require('charwise')
const { EntryStream } = require('level-read-stream')
const levelOptions = require('./options.js')
// const tokenParser = require('./parseToken.js')
// const charwise = require('charwise')
// const { EntryStream } = require('level-read-stream')
// const levelOptions = require('./options.js')

import tokenParser from './parseToken.js'
import charwise from 'charwise'
import { EntryStream } from 'level-read-stream'
import levelOptions from './options.js'

// polyfill- HI and LO coming in next version of charwise
charwise.LO = null
charwise.HI = undefined

module.exports = ops => {
// module.exports = ops => {

export default function (ops) {
const isString = s => typeof s === 'string'

// TODO: in order to account for query processing pipelines,
Expand Down Expand Up @@ -56,7 +63,7 @@ module.exports = ops => {
// If this token is a stopword then return 'undefined'
const removeStopwords = token =>
token.VALUE.GTE === token.VALUE.LTE &&
ops.stopwords.includes(token.VALUE.GTE)
ops.stopwords.includes(token.VALUE.GTE)
? undefined
: token

Expand Down Expand Up @@ -84,7 +91,6 @@ module.exports = ops => {
// testForBreak(token) // ?
token = await queryReplace(token) // TODO: rename to replaceToken?
testForBreak(token)

token = await pipeline(token)
testForBreak(token)
} catch (e) {
Expand Down Expand Up @@ -201,7 +207,9 @@ module.exports = ops => {
// documents exist in the index.
const EXIST = (...ids) =>
Promise.all(
ids.map(id => ops._db.get([ops.docExistsSpace, id], levelOptions).catch(e => null))
ids.map(id =>
ops._db.get([ops.docExistsSpace, id], levelOptions).catch(e => null)
)
).then(result =>
result.reduce((acc, cur, i) => {
if (cur != null) acc.push(ids[i])
Expand Down Expand Up @@ -250,7 +258,9 @@ module.exports = ops => {

const OBJECT = _ids =>
Promise.all(
_ids.map(id => ops._db.get(['DOC', id._id], levelOptions).catch(reason => null))
_ids.map(id =>
ops._db.get(['DOC', id._id], levelOptions).catch(reason => null)
)
).then(_objects =>
_ids.map((_id, i) => {
_id._object = _objects[i]
Expand Down
29 changes: 19 additions & 10 deletions src/write.js
Original file line number Diff line number Diff line change
@@ -1,8 +1,12 @@
const trav = require('traverse')
const reader = require('./read.js')
const levelOptions = require('./options.js')
// const trav = require('traverse')
// const reader = require('./read.js')
// const levelOptions = require('./options.js')

module.exports = ops => {
import trav from 'traverse'
import reader from './read.js'
import levelOptions from './options.js'

export default function (ops) {
// TODO: set reset this to the max value every time the DB is restarted
let incrementalId = 0

Expand Down Expand Up @@ -202,11 +206,12 @@ module.exports = ops => {
// when importing, index is first cleared. This means that "merges"
// are not currently supported
const IMPORT = index =>
ops._db
.clear()
.then(() =>
ops._db.batch(index.map(entry => Object.assign(entry, { type: 'put' })), levelOptions)
ops._db.clear().then(() =>
ops._db.batch(
index.map(entry => Object.assign(entry, { type: 'put' })),
levelOptions
)
)

const PUT = (docs, putOptions = {}) =>
writer(
Expand All @@ -221,14 +226,18 @@ module.exports = ops => {
).then(TIMESTAMP_LAST_UPDATED)

const TIMESTAMP_LAST_UPDATED = passThrough =>
ops._db.put(['~LAST_UPDATED'], Date.now(), levelOptions).then(() => passThrough)
ops._db
.put(['~LAST_UPDATED'], Date.now(), levelOptions)
.then(() => passThrough)

const TIMESTAMP_CREATED = () =>
ops._db
.get(['~CREATED'], levelOptions)
.then(/* already created- do nothing */)
.catch(e =>
ops._db.put(['~CREATED'], Date.now(), levelOptions).then(TIMESTAMP_LAST_UPDATED)
ops._db
.put(['~CREATED'], Date.now(), levelOptions)
.then(TIMESTAMP_LAST_UPDATED)
)

return {
Expand Down
8 changes: 5 additions & 3 deletions test/src/AGGREGATION_FILTER-test.js
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
const fii = require('../../')
const test = require('tape')
const { InvertedIndex } = await import(
'../../src/' + process.env.FII_ENTRYPOINT
)
import test from 'tape'

const sandbox = 'test/sandbox/'
const indexName = sandbox + 'AGGREGATION_FILTER'
Expand Down Expand Up @@ -99,7 +101,7 @@ const data = [

test('create index', t => {
t.plan(1)
fii({ name: indexName }).then(db => {
new InvertedIndex({ name: indexName }).then(db => {
global[indexName] = db
t.ok(db, !undefined)
})
Expand Down
8 changes: 5 additions & 3 deletions test/src/AND-test.js
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
const fii = require('../../')
const test = require('tape')
const { InvertedIndex } = await import(
'../../src/' + process.env.FII_ENTRYPOINT
)
import test from 'tape'

const sandbox = 'test/sandbox/'
const indexName = sandbox + 'AND'
Expand Down Expand Up @@ -99,7 +101,7 @@ const data = [

test('create index', t => {
t.plan(1)
fii({ name: indexName }).then(db => {
new InvertedIndex({ name: indexName }).then(db => {
global[indexName] = db
t.ok(db, !undefined)
})
Expand Down
Loading

0 comments on commit 1bb22a4

Please sign in to comment.