Skip to content

Commit

Permalink
fix(nlu): crfsuite is silent
Browse files Browse the repository at this point in the history
  • Loading branch information
slvnperron committed Dec 12, 2018
1 parent f6a0eb6 commit 514ab71
Show file tree
Hide file tree
Showing 4 changed files with 11 additions and 6 deletions.
17 changes: 11 additions & 6 deletions modules/nlu/src/backend/pipelines/slots/crf_extractor.ts
Expand Up @@ -16,7 +16,7 @@ const CRF_TRAINER_PARAMS = {
c2: '0.01',
max_iterations: '500',
'feature.possible_transitions': '1',
'feature.possible_states': '1',
'feature.possible_states': '1'
}

type Features = Partial<{
Expand Down Expand Up @@ -109,8 +109,11 @@ export default class CRFExtractor implements SlotExtractor {

private async _trainCrf(samples: Token[][]) {
this._crfModelFn = tmp.fileSync({ postfix: '.bin' }).name
const trainer = Trainer()
const trainer = new Trainer()
trainer.set_params(CRF_TRAINER_PARAMS)
trainer.set_callback(str => {
/* swallow training results */
})

for (const sample of samples) {
const entries: string[][] = []
Expand All @@ -132,10 +135,12 @@ export default class CRFExtractor implements SlotExtractor {
this._ft = new FastText(this._ftModelFn)

const trainContent = samples.reduce((corpus, example) => {
const cannonicSentence = example.map(s => {
if (s.type === 'o') return s.value
else return s.type.slice(2)
}).join(' ') // TODO replace this by the opposite tokenizer as only latin language use \s...
const cannonicSentence = example
.map(s => {
if (s.type === 'o') return s.value
else return s.type.slice(2)
})
.join(' ') // TODO replace this by the opposite tokenizer as only latin language use \s...
return `${corpus}${cannonicSentence}}\n`
}, '')

Expand Down
Empty file modified modules/nlu/src/backend/tools/bin/crfsuite_linux.node 100644 → 100755
Empty file.
Binary file modified modules/nlu/src/backend/tools/bin/crfsuite_osx.node
Binary file not shown.
Binary file modified modules/nlu/src/backend/tools/bin/crfsuite_win.node
Binary file not shown.

0 comments on commit 514ab71

Please sign in to comment.