Merge pull request #73 from stanford-oval/wip/update-deps

Update to thingtalk 1.8.0-beta.1, thingpedia 2.5.0-beta.1

.travis.yml

@@ -10,7 +10,6 @@ before_install:
 - sudo apt-get install -y realpath
 before_script:
 - greenkeeper-lockfile-update
-- (cd node_modules/thingtalk ; yarn)
 after_script: greenkeeper-lockfile-upload
 script:
 - yarn lint

package.json

@@ -9,20 +9,19 @@
   "bin": {
     "genie": "./tool/genie.js"
   },
-  "resolutions": {
-    "thingtalk": "github:stanford-oval/thingtalk"
-  },
+  "resolutions": {},
   "dependencies": {
     "argparse": "^1.0.10",
     "byline": "^5.0.0",
-    "csv": "^5.0.0",
+    "csv-parse": "^4.4.5",
+    "csv-stringify": "5.3.0",
     "mmap-io": "^1.1.7",
     "qs": "^6.7.0",
     "seedrandom": "^3.0.0",
     "shuffle-array": "^1.0.1",
     "sockaddr": "^1.0.1",
-    "thingpedia": "^2.3.3",
-    "thingtalk": "github:stanford-oval/thingtalk"
+    "thingpedia": "~2.5.0-beta.1",
+    "thingtalk": "~1.8.0-beta.1"
   },
   "devDependencies": {
     "coveralls": "^3.0.0",

tool/contextual-prepare-turk.js

@@ -12,7 +12,7 @@
 const Stream = require('stream');
 const seedrandom = require('seedrandom');
 const fs = require('fs');
-const csv = require('csv');
+const csvstringify = require('csv-stringify');
 
 const ThingTalk = require('thingtalk');
 const Type = ThingTalk.Type;
@@ -307,7 +307,7 @@ module.exports = {
                 }
             }))
             .pipe(new SentenceSampler(constants, options))
-            .pipe(csv.stringify({ header: true, delimiter: ',' }))
+            .pipe(csvstringify({ header: true, delimiter: ',' }))
             .pipe(args.output);
 
         return new Promise((resolve, reject) => {

tool/evaluate-file.js

@@ -10,7 +10,7 @@
 "use strict";
 
 const fs = require('fs');
-const csv = require('csv');
+const csvparse = require('csv-parse');
 const byline = require('byline');
 const Stream = require('stream');
 const ThingTalk = require('thingtalk');
@@ -78,7 +78,7 @@ module.exports = {
             ['id', 'context', 'sentence', 'target_code', 'prediction'] :
             ['id', 'sentence', 'target_code', 'prediction'];
         const predictionstream = args.predictions
-            .pipe(csv.parse({ columns, delimiter: '\t', relax: true }))
+            .pipe(csvparse({ columns, delimiter: '\t', relax: true }))
             .pipe(new StreamUtils.MapAccumulator());
         const predictions = await predictionstream.read();
 

tool/lib/file_parameter_provider.js

@@ -12,7 +12,7 @@
 const util = require('util');
 const fs = require('fs');
 const byline = require('byline');
-const csv = require('csv');
+const csvparse = require('csv-parse');
 const path = require('path');
 
 // Load strings and entities from files
@@ -64,7 +64,7 @@ module.exports = class FileParameterProvider {
 
         const strings = [];
         const input = fs.createReadStream(filepath)
-            .pipe(csv.parse({ delimiter: '\t', relax: true }));
+            .pipe(csvparse({ delimiter: '\t', relax: true }));
 
         input.on('data', (line) => {
             let preprocessed, weight;

tool/manual-annotate.js

@@ -11,7 +11,7 @@
 
 const fs = require('fs');
 const events = require('events');
-const csv = require('csv');
+const csvparse = require('csv-parse');
 const readline = require('readline');
 
 const ParserClient = require('./lib/parserclient');
@@ -276,7 +276,7 @@ module.exports = {
 
         let lines = [];
         args.input.setEncoding('utf8');
-        const input = args.input.pipe(csv.parse({ columns: true, relax: true, delimiter: '\t' }));
+        const input = args.input.pipe(csvparse({ columns: true, relax: true, delimiter: '\t' }));
         input.on('data', (line) => {
             lines.push(line);
         });

tool/mturk-make-paraphrase-hits.js

@@ -11,7 +11,8 @@
 
 const fs = require('fs');
 const Stream = require('stream');
-const csv = require('csv');
+const csvparse = require('csv-parse');
+const csvstringify = require('csv-stringify');
 
 const StreamUtils = require('../lib/stream-utils');
 const { NUM_SENTENCES_PER_TASK } = require('./lib/constants');
@@ -84,9 +85,9 @@ module.exports = {
     async execute(args) {
         process.stdin.setEncoding('utf8');
         process.stdin
-            .pipe(csv.parse({ columns: true, delimiter: '\t' }))
+            .pipe(csvparse({ columns: true, delimiter: '\t' }))
             .pipe(new ParaphraseHITCreator(args.sentences_per_task))
-            .pipe(csv.stringify({ header: true, delimiter: ',' }))
+            .pipe(csvstringify({ header: true, delimiter: ',' }))
             .pipe(args.output);
 
         return StreamUtils.waitFinish(args.output);

tool/mturk-make-validation-hits.js

@@ -12,7 +12,8 @@
 
 const fs = require('fs');
 const Stream = require('stream');
-const csv = require('csv');
+const csvparse = require('csv-parse');
+const csvstringify = require('csv-stringify');
 const seedrandom = require('seedrandom');
 const shuffle = require('shuffle-array');
 const ThingTalk = require('thingtalk');
@@ -195,7 +196,7 @@ module.exports = {
         // so we can sample a fake one to choose
 
         const accumulator = new ArrayAccumulator();
-        process.stdin.pipe(csv.parse({
+        process.stdin.pipe(csvparse({
                 columns: true,
                 delimiter: ',',
                 relax_column_count: true
@@ -220,7 +221,7 @@ module.exports = {
                 debug: args.debug,
                 rng: rng
             }))
-            .pipe(csv.stringify({ header: true, delimiter: ',' }))
+            .pipe(csvstringify({ header: true, delimiter: ',' }))
             .pipe(args.output);
 
         await waitFinish(args.output);

tool/mturk-validate.js

@@ -11,7 +11,8 @@
 
 const fs = require('fs');
 const Stream = require('stream');
-const csv = require('csv');
+const csvparse = require('csv-parse');
+const csvstringify = require('csv-stringify');
 const ThingTalk = require('thingtalk');
 
 const { ParaphraseValidatorFilter } = require('../lib/validator');
@@ -120,7 +121,7 @@ module.exports = {
             if (!args.validation_input)
                 throw new Error(`Argument --validation-input is required when performing manual validation`);
             const validationInput = fs.createReadStream(args.validation_input)
-                .pipe(csv.parse({
+                .pipe(csvparse({
                     columns: true,
                     delimiter: ',',
                     relax_column_count: true
@@ -131,7 +132,7 @@ module.exports = {
 
             if (args.validation_rejects) {
                 validationRejects = StreamUtils.waitFinish(validationInput
-                    .pipe(csv.stringify({ header: true, delimiter: ',' }))
+                    .pipe(csvstringify({ header: true, delimiter: ',' }))
                     .pipe(fs.createWriteStream(args.validation_rejects)));
             }
 
@@ -148,7 +149,7 @@ module.exports = {
         }
 
         const rejectedPara = fs.createReadStream(args.paraphrasing_input)
-            .pipe(csv.parse({
+            .pipe(csvparse({
                 columns: true,
                 delimiter: ',',
                 relax_column_count: true
@@ -163,7 +164,7 @@ module.exports = {
         let paraphrasingRejects;
         if (args.paraphrasing_rejects) {
             paraphrasingRejects = StreamUtils.waitFinish(rejectedPara
-                .pipe(csv.stringify({ header: true, delimiter: ',' }))
+                .pipe(csvstringify({ header: true, delimiter: ',' }))
                 .pipe(fs.createWriteStream(args.paraphrasing_rejects)));
         } else {
             paraphrasingRejects = Promise.resolve();

tool/sample.js

@@ -12,7 +12,7 @@
 const seedrandom = require('seedrandom');
 const fs = require('fs');
 const byline = require('byline');
-const csv = require('csv');
+const csvstringify = require('csv-stringify');
 const Stream = require('stream');
 const ThingTalk = require('thingtalk');
 
@@ -293,7 +293,7 @@ module.exports = {
         readAllLines(args.input_file)
             .pipe(new DatasetParser({ contextual: args.contextual, preserveId: true }))
             .pipe(new SentenceSampler(schemaRetriever, constants, options))
-            .pipe(csv.stringify({ header: true, delimiter: '\t' }))
+            .pipe(csvstringify({ header: true, delimiter: '\t' }))
             .pipe(args.output);
 
         return new Promise((resolve, reject) => {

yarn.lock

@@ -342,9 +342,9 @@ core-util-is@1.0.2, core-util-is@~1.0.0:
   integrity sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=
 
 coveralls@^3.0.0:
-  version "3.0.5"
-  resolved "https://registry.yarnpkg.com/coveralls/-/coveralls-3.0.5.tgz#28d7274c6c9827aa85537eab82d66e7e62d0d527"
-  integrity sha512-/KD7PGfZv/tjKB6LoW97jzIgFqem0Tu9tZL9/iwBnBd8zkIZp7vT1ZSHNvnr0GSQMV/LTMxUstWg8WcDDUVQKg==
+  version "3.0.6"
+  resolved "https://registry.yarnpkg.com/coveralls/-/coveralls-3.0.6.tgz#5c63b2759b6781118e7439bd870ba5e9ee428b25"
+  integrity sha512-Pgh4v3gCI4T/9VijVrm8Ym5v0OgjvGLKj3zTUwkvsCiwqae/p6VLzpsFNjQS2i6ewV7ef+DjFJ5TSKxYt/mCrA==
   dependencies:
     growl "~> 1.10.0"
     js-yaml "^3.13.1"
@@ -383,33 +383,18 @@ cross-spawn@^6.0.5:
     shebang-command "^1.2.0"
     which "^1.2.9"
 
-csv-generate@^3.2.0:
-  version "3.2.3"
-  resolved "https://registry.yarnpkg.com/csv-generate/-/csv-generate-3.2.3.tgz#24004f21de61c2ea1c4474d3e65a18261f638a80"
-  integrity sha512-IcR3K0Nx+nJAkcU2eAglVR7DuHnxcuhUM2w2cR+aHOW7bZp2S5LyN2HF3zTkp6BV/DjR6ykoKznUm+AjnWcOKg==
+csv-parse@^4.4.5:
+  version "4.4.5"
+  resolved "https://registry.yarnpkg.com/csv-parse/-/csv-parse-4.4.5.tgz#383255357a766c220be049e4f1f2c3016eb9ac66"
+  integrity sha512-koPV9m9AjNJCK3ig4ErgRJalZsLxWI7NP0Fd3+CO9hgDZt3FSljTeESnfWTbyRc8qk/3/LgX1s5naDqLxiuK9w==
 
-csv-parse@^4.3.0:
-  version "4.4.4"
-  resolved "https://registry.yarnpkg.com/csv-parse/-/csv-parse-4.4.4.tgz#57039751ae415d1404ce2174f422c7ae9a54d7c1"
-  integrity sha512-AjJHtVpO7/z5QywbCrDsuMMzeFC9ReZQGojveqfohzOLUXHgujqjPIlxOgY3KAboqwL8azuuUajxcDblYEGo4g==
-
-csv-stringify@^5.1.2:
+csv-stringify@5.3.0:
   version "5.3.0"
   resolved "https://registry.yarnpkg.com/csv-stringify/-/csv-stringify-5.3.0.tgz#ff2dfafa6fcccd455ff5039be9c202475aa3bbe0"
   integrity sha512-VMYPbE8zWz475smwqb9VbX9cj0y4J0PBl59UdcqzLkzXHZZ8dh4Rmbb0ZywsWEtUml4A96Hn7Q5MW9ppVghYzg==
   dependencies:
     lodash.get "~4.4.2"
 
-csv@^5.0.0:
-  version "5.1.1"
-  resolved "https://registry.yarnpkg.com/csv/-/csv-5.1.1.tgz#bd715f15a4ed9141309a24ec6dab1c903ab79f92"
-  integrity sha512-gezB9D+enrh2tLj+vsAD8JyYRMIJdSMpec/Pgbb+7YRj6Q6/D12HLSwjhx+CrirRT4dESjZYXWX1JfqlV4RlTA==
-  dependencies:
-    csv-generate "^3.2.0"
-    csv-parse "^4.3.0"
-    csv-stringify "^5.1.2"
-    stream-transform "^1.0.8"
-
 dashdash@^1.12.0:
   version "1.14.1"
   resolved "https://registry.yarnpkg.com/dashdash/-/dashdash-1.14.1.tgz#853cfa0f7cbe2fed5de20326b8dd581035f6e2f0"
@@ -1334,11 +1319,6 @@ os-tmpdir@~1.0.2:
   resolved "https://registry.yarnpkg.com/os-tmpdir/-/os-tmpdir-1.0.2.tgz#bbe67406c79aa85c5cfec766fe5734555dfa1274"
   integrity sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ=
 
-p-finally@^1.0.0:
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/p-finally/-/p-finally-1.0.0.tgz#3fbcfb15b899a44123b34b6dcc18b724336a2cae"
-  integrity sha1-P7z7FbiZpEEjs0ttzBi3JDNqLK4=
-
 p-limit@^2.0.0:
   version "2.2.0"
   resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-2.2.0.tgz#417c9941e6027a9abcba5092dd2904e255b5fbc2"
@@ -1627,9 +1607,9 @@ sax@>=0.6.0, sax@^1.2.4:
   integrity sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==
 
 seedrandom@^3.0.0:
-  version "3.0.1"
-  resolved "https://registry.yarnpkg.com/seedrandom/-/seedrandom-3.0.1.tgz#eb3dde015bcf55df05a233514e5df44ef9dce083"
-  integrity sha512-1/02Y/rUeU1CJBAGLebiC5Lbo5FnB22gQbIFFYTLkwvp1xdABZJH1sn4ZT1MzXmPpzv+Rf/Lu2NcsLJiK4rcDg==
+  version "3.0.3"
+  resolved "https://registry.yarnpkg.com/seedrandom/-/seedrandom-3.0.3.tgz#74efcc324533b417b8f3e2cf2a70797aa4a94441"
+  integrity sha512-PJLhhxIMjlMJaiIRtqiVW061EZn3cS+waZkbFe7eCa2R3g88HbNdWmw4NTFG1w5unxd0GeNaUUxZJP7gPAzSDQ==
 
 "semver@2 || 3 || 4 || 5", semver@^5.5.0, semver@^5.6.0:
   version "5.7.0"
@@ -1757,11 +1737,6 @@ sshpk@^1.7.0:
     safer-buffer "^2.0.2"
     tweetnacl "~0.14.0"
 
-stream-transform@^1.0.8:
-  version "1.0.8"
-  resolved "https://registry.yarnpkg.com/stream-transform/-/stream-transform-1.0.8.tgz#54f721122d310eca855a16c97939881ab5bbb76c"
-  integrity sha512-1q+dL790Ps0NV33rISMq9OLtfDA9KMJZdo1PHZXE85orrWsM4FAh8CVyAOTHO0rhyeM138KNPngBPrx33bFsxw==
-
 string-width@^2.1.0:
   version "2.1.1"
   resolved "https://registry.yarnpkg.com/string-width/-/string-width-2.1.1.tgz#ab93f27a8dc13d28cac815c462143a6d9012ae9e"
@@ -1849,22 +1824,22 @@ text-table@^0.2.0:
   resolved "https://registry.yarnpkg.com/text-table/-/text-table-0.2.0.tgz#7f5ee823ae805207c00af2df4a84ec3fcfa570b4"
   integrity sha1-f17oI66AUgfACvLfSoTsP8+lcLQ=
 
-thingpedia@^2.3.3:
-  version "2.4.0"
-  resolved "https://registry.yarnpkg.com/thingpedia/-/thingpedia-2.4.0.tgz#a4bd953848e87fb55dfcdc56e231810c22b58bcc"
-  integrity sha512-AVgQwYBH16pU7OcE5mhsFs3oHZ9EsJG0mqGohNZWLVEHsdiWxoahaMVRQFypXpxlcyF/ksCQ8I/0U3qWmdDJXQ==
+thingpedia@~2.5.0-beta.1:
+  version "2.5.0-beta.1"
+  resolved "https://registry.yarnpkg.com/thingpedia/-/thingpedia-2.5.0-beta.1.tgz#0c82703d17d588c52398b897f10b293d2cd9f32d"
+  integrity sha512-/nrynhUjHWORLgumnnV4KihBsigx6Cf/OsvCEnSfyDvoDs4H4Bz+rHObVZYmI6mRoGzkgbILn0AfeZopJeNB7Q==
   dependencies:
     feedparser "^2.2.9"
     ip "~1.1.0"
     oauth "~0.9.14"
-    p-finally "^1.0.0"
-    thingtalk "^1.7.0"
+    thingtalk "~1.8.0-beta.1"
+    tmp "^0.1.0"
     xml2js "^0.4.17"
 
-thingtalk@^1.7.0, "thingtalk@github:stanford-oval/thingtalk":
-  version "1.7.3"
-  uid "332f5176f6107f5d5b581841041cc8d33efcebc3"
-  resolved "https://codeload.github.com/stanford-oval/thingtalk/tar.gz/332f5176f6107f5d5b581841041cc8d33efcebc3"
+thingtalk@~1.8.0-beta.1:
+  version "1.8.0-beta.1"
+  resolved "https://registry.yarnpkg.com/thingtalk/-/thingtalk-1.8.0-beta.1.tgz#e3db697173b60f6eb160016ce11743ffe1735fc5"
+  integrity sha512-ozn0gbBcKfNCXyS/zsUsVrwCWYwmLNdHAwYXHAOQmcKBNflKt/dWGp5gJvmt+zX9uahN5o33St4sItJSRQjTpg==
   dependencies:
     adt "~0.7.2"
     byline "^5.0.0"
@@ -1883,6 +1858,13 @@ tmp@^0.0.33:
   dependencies:
     os-tmpdir "~1.0.2"
 
+tmp@^0.1.0:
+  version "0.1.0"
+  resolved "https://registry.yarnpkg.com/tmp/-/tmp-0.1.0.tgz#ee434a4e22543082e294ba6201dcc6eafefa2877"
+  integrity sha512-J7Z2K08jbGcdA1kkQpJSqLF6T0tdQqpR2pnSUXsIchbPdTI9v3e85cLW0d6WDhwuAleOV71j2xWs8qMPfK7nKw==
+  dependencies:
+    rimraf "^2.6.3"
+
 to-fast-properties@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/to-fast-properties/-/to-fast-properties-2.0.0.tgz#dc5e698cbd079265bc73e0377681a4e4e83f616e"