Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

added io-specific, slow test folder

  • Loading branch information...
commit 893a52961780788d0b4d17a1c071e596c12ed914 1 parent dbe1e94
@chrisumbel chrisumbel authored
View
10 Makefile
@@ -21,9 +21,15 @@
benchmark:
@node benchmarks
-clean:
+test_clean:
+ @ls ./spec/test_data/wordnet/download/* | xargs rm -f
+
+clean: test_clean
@find ./ -name *~ | xargs rm -f
@find ./ -name \#* | xargs rm -f
- @rm ./spec/test_data/wordnet/*
+
test:
@NODE_PATH=. jasmine-node spec/
+
+test_io: test_clean
+ @NODE_PATH=. jasmine-node io_spec/
View
21 lib/natural/wordnet/file_searcher.js → io_spec/wordnet_spec.js
@@ -1,4 +1,4 @@
-/*
+/*
Copyright (c) 2011, Chris Umbel
Permission is hereby granted, free of charge, to any person obtaining a copy
@@ -20,3 +20,22 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
+var Wordnet = require('lib/natural/wordnet/wordnet');
+jasmine.asyncSpecWait.timeout = 30 * 1000;
+
+describe('wordnet', function() {
+ it('should download files then', function() {
+ var wordnet = new Wordnet('./spec/test_data/wordnet/download/', 'http://wordnet.naturalnode.com/');
+
+ wordnet.lookup('entity', function(records) {
+ expect(records.length).toBe(1);
+ expect(records[0].word).toBe('entity');
+ require('path').exists('./spec/test_data/wordnet/download/index.noun', function(exists) {
+ expect(exists).toBeTruthy();
+ asyncSpecDone();
+ });
+ });
+
+ asyncSpecWait();
+ });
+});
View
66 lib/natural/wordnet/wordnet_file.js
@@ -20,7 +20,48 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
-var fs = require('fs');
+var compress = require('compress'),
+ gzip = new compress.Gunzip()
+ fs = require('fs'),
+ path = require('path'),
+ url = require('url');
+
+function downloadFile(url, filePath, callback) {
+ gzip.init();
+
+ var http = require('http');
+ var client = http.createClient(80, url.host);
+ var request = client.request('GET', url.pathname, {host: url.host});
+
+ client.on('error', function(err) {
+ callback();
+ });
+
+ request.end();
+
+ fs.open(filePath, 'w', null, function(err, outFd) {
+ request.on('response', function(response) {
+ var size = parseInt(response.headers['content-length']);
+ var progress = 0;
+
+ response.on('data', function(data) {
+ var rawLen = data.length;
+ data = gzip.inflate(data.toString('binary'), 'binary');
+ var outBuff = new Buffer(data.length);
+ outBuff.write(data.toString(), 0, 'binary');
+
+ fs.write(outFd, outBuff, 0, outBuff.length, null, function(err, count) {
+ progress += rawLen;
+
+ if(progress >= size) {
+ fs.close(outFd);
+ callback();
+ };
+ });
+ });
+ });
+ });
+}
function appendLineChar(fd, pos, buffPos, buff, callback) {
fs.read(fd, buff, buffPos, 1, pos, function(err, count) {
@@ -35,16 +76,31 @@ function appendLineChar(fd, pos, buffPos, buff, callback) {
});
}
-function open(callback) {
- fs.open(this.filePath, 'r', null, function(err, fd) {
- callback(err, fd);
+function open(callback) {
+ var filePath = this.filePath;
+ var url = this.url;
+
+ path.exists(filePath, function(exists) {
+ var _open = function(){
+ fs.open(filePath, 'r', null, function(err, fd) {
+ callback(err, fd);
+ });
+ };
+
+ if(exists) {
+ _open();
+ } else {
+ downloadFile(url, filePath, function() {
+ _open();
+ })
+ }
});
}
var WordnetFile = function(dataDir, base, fileName) {
this.dataDir = dataDir;
this.fileName = fileName;
- this.url = require('url').resolve(base, this.fileName);
+ this.url = url.parse(url.resolve(base, this.fileName + '.gz'));
this.filePath = require('path').join(this.dataDir, this.fileName);
this.open = open;
};
View
13 spec/test_data/search_file
@@ -1,13 +0,0 @@
-A 1
-B 2
-C 3
-D 4
-E 5
-F 6
-G 7
-H 8
-I 9
-J 10
-K 11
-L 12
-M 13
View
9 spec/wordnet_spec.js
@@ -20,8 +20,7 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
-var fileSearcher = require('lib/natural/wordnet/file_searcher'),
- IndexFile = require('lib/natural/wordnet/index_file'),
+var IndexFile = require('lib/natural/wordnet/index_file'),
DataFile = require('lib/natural/wordnet/data_file'),
Wordnet = require('lib/natural/wordnet/wordnet');
@@ -29,9 +28,9 @@ describe('wordnet', function() {
describe('index_file', function() {
it('should build a valid url', function() {
var indexFile = new IndexFile('./spec/test_data/wordnet', 'http://wordnet.naturalnode.com/', 'noun');
- expect(indexFile.url).toBe('http://wordnet.naturalnode.com/index.noun');
+ expect(indexFile.url.href).toBe('http://wordnet.naturalnode.com/index.noun.gz');
});
-
+
it('should miss a record', function() {
var indexFile = new IndexFile('./spec/test_data/wordnet', 'http://wordnet.naturalnode.com/', 'noun');
indexFile.lookup('aac', function(result) {
@@ -80,5 +79,5 @@ describe('wordnet', function() {
});
asyncSpecWait();
- });
+ });
});
Please sign in to comment.
Something went wrong with that request. Please try again.