Permalink
Browse files

license headers and a lot of documentation

  • Loading branch information...
1 parent aeb64e5 commit da1016ae0ab8908591aca87121e5f882b4629a13 @4poc committed Feb 28, 2011
Showing with 394 additions and 138 deletions.
  1. +8 −4 feedability.js
  2. +30 −38 lib/cache.js
  3. +29 −3 lib/ce.js
  4. +26 −1 lib/cfg.js
  5. +21 −2 lib/entity.js
  6. +25 −8 lib/feed.js
  7. +29 −4 lib/filter.js
  8. +37 −10 lib/func.js
  9. +33 −3 lib/log.js
  10. +126 −9 lib/proxy.js
  11. +30 −19 lib/urlopen.js
  12. +0 −37 test/urlopen.js
View
@@ -1,5 +1,5 @@
/**
- * Feedability: NodeJS Feed Proxy With Readability
+ * Feedability: Node.js Feed Proxy With Readability
* Copyright (c) 2011, Matthias -apoc- Hecker <http://apoc.cc/>
*
* This program is free software: you can redistribute it and/or modify
@@ -17,19 +17,23 @@
*/
console.log('Starting Feedability: NodeJS Feed Proxy With Readability\n');
+/**
+ * Core module for startup and shutdown of the feedability HTTP server.
+ *
+ * @fileOverview
+ */
+
// built in libraries
var fs = require('fs'),
http = require('http'),
util = require('util');
// internal libraries
var cfg = require('./lib/cfg.js'),
+ log = new (require('./lib/log.js').Logger)('core'),
func = require('./lib/func.js'),
ProxyRequest = require('./lib/proxy.js').ProxyRequest;
-// create local logging object
-var log = new (require('./lib/log.js').Logger)('core');
-
// some variables used for the http server
var bind = cfg.get('proxy')['bind'];
var port = cfg.get('proxy')['port'];
View
@@ -1,44 +1,36 @@
-var log = new (require('./log.js').Logger)('cache');
-
-/*
-var uri = require('url'),
- fs = require('fs');
-var func = require('./func.js'),
- cfg = require('./cfg.js');
-
-// returns the name of the cache file for the supplied url
-function filename(ext, url)
-{
- var domain = uri.parse(url).hostname,
- urlhash = func.sha1(url);
- var cache_path = cfg.get('cache')['path']+'/'+domain;
-
- if(!func.file_exists(cache_path)) {
- log.info('create non-existing domain directory', domain);
- fs.mkdirSync(cache_path, 0755);
- }
-
- return cache_path + '/' + urlhash + '.' + ext;
-}
-exports.filename = filename;
-
-// make sure the caching directory exists, if not create it (sync)
-exports.create_path = function() {
- var cache_path = cfg.get('cache')['path'];
- if(!func.file_exists(cache_path)) {
- log.info('create cache directory: '+cache_path);
- fs.mkdirSync(cache_path, 0755);
- }
-};
-*/
-
-
-
-
+/**
+ * Feedability: Node.js Feed Proxy With Readability
+ * Copyright (c) 2011, Matthias -apoc- Hecker <http://apoc.cc/>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/**
+ * Contains the class for cache accessing, it is used to store
+ * received feeds and pages by the urlopen module and to cache the
+ * content extracted by readability.
+ *
+ * @fileOverview
+ */
+
+// built in libraries
var fs = require('fs'),
uri = require('url');
-var func = require('./func.js'),
+// internal libraries
+var log = new (require('./log.js').Logger)('cache'),
+ func = require('./func.js'),
cfg = require('./cfg.js');
var Cache = function(url, extension) {
View
@@ -1,17 +1,43 @@
-var log = new (require('./log.js').Logger)('ce');
+/**
+ * Feedability: Node.js Feed Proxy With Readability
+ * Copyright (c) 2011, Matthias -apoc- Hecker <http://apoc.cc/>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/**
+ * Contains a class for content extraction based on arc90's
+ * Readability. This is the place where future development of
+ * other content extraction algorithms and/or template detection
+ * systems _could_ be developed.
+ *
+ * @fileOverview
+ */
// built in libraries
var fs = require('fs'),
uri = require('url');
// internal libraries
-var func = require('./func.js'),
+var log = new (require('./log.js').Logger)('ce'),
+ func = require('./func.js'),
cfg = require('./cfg.js'),
Filter = require('./filter.js').Filter,
Cache = require('./cache.js').Cache,
urlopen = require('./urlopen.js');
-
+// external libraries
var readability = require('readability');
View
@@ -1,8 +1,33 @@
/**
- * Module to load and access settings.
+ * Feedability: Node.js Feed Proxy With Readability
+ * Copyright (c) 2011, Matthias -apoc- Hecker <http://apoc.cc/>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
+
+/**
+ * Module provides methods for accessing the json based configuration
+ * settings.
+ *
+ * @fileOverview
+ */
+
+// built in libraries
var fs = require('fs'),
util = require('util');
+
+// external libraries
var func = require('./func.js');
var settings = null;
View
@@ -1,7 +1,26 @@
/**
- * XML and HTML Entity Encoding/Decoding
+ * Feedability: Node.js Feed Proxy With Readability
+ * Copyright (c) 2011, Matthias -apoc- Hecker <http://apoc.cc/>
*
- * @author Matthias -apoc- Hecker
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/**
+ * Provide methods for XML and HTML entity encoding and decoding based
+ * on w3c's XML and HTML 4 specifications.
+ *
+ * @fileOverview
*/
/**
View
@@ -1,4 +1,22 @@
/**
+ * Feedability: Node.js Feed Proxy With Readability
+ * Copyright (c) 2011, Matthias -apoc- Hecker <http://apoc.cc/>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/**
* Contains the feed class for parsing and generating the XML based
* feeds. Atom and RSS feeds are supported. Its not meant to be a
* complete implementation of RSS or Atom, thats the job of the
@@ -7,21 +25,20 @@
* @fileOverview
*/
-var log = new (require('./log.js').Logger)('feed');
-
// built-in libraries
var util = require('util');
-// external libraries
-var expat = require('node-expat');
-
// internal libraries
-var func = require('./func.js'),
+var log = new (require('./log.js').Logger)('feed'),
+ func = require('./func.js'),
cfg = require('./cfg.js'),
entity = require('./entity.js'),
cache = require('./cache.js'),
urlopen = require('./urlopen.js');
+// external libraries
+var expat = require('node-expat');
+
/**
* # Feedability Feed Parser/Generator Class (Constructor)
*
@@ -298,8 +315,8 @@ Feed.prototype.expatEndElement = function(name) {
Feed.prototype.expatText = function(text) {
log.debug('expat stream event: text, text length: '+text.length);
if(!this.cdata_text) {
- // decode the xml entities of non-cdata text nodes:
- text = entity.decode(text);
+ // encode the xml entities of non-cdata text nodes:
+ text = entity.encode(text, {numbered: true});
}
if(this.prev_open) this.stream += '>';
View
@@ -1,18 +1,43 @@
-var log = new (require('./log.js').Logger)('filter');
+/**
+ * Feedability: Node.js Feed Proxy With Readability
+ * Copyright (c) 2011, Matthias -apoc- Hecker <http://apoc.cc/>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/**
+ * Module provide filtering class for pre and post processing of
+ * feed, to be more precise: the dom and regular expression based
+ * manipulation of single received pages/ feed items.
+ *
+ * @fileOverview
+ */
// built in libraries
var fs = require('fs'),
uri = require('url'),
util = require('util');
-var jsdom = require('jsdom');
-
// internal libraries
-var func = require('./func.js'),
+var log = new (require('./log.js').Logger)('filter'),
+ func = require('./func.js'),
cfg = require('./cfg.js'),
Cache = require('./cache.js').Cache,
urlopen = require('./urlopen.js');
+// external libraries
+var jsdom = require('jsdom');
var Filter = function(url) {
View
@@ -1,20 +1,47 @@
/**
- * Node.js Utility Functions
+ * Feedability: Node.js Feed Proxy With Readability
+ * Copyright (c) 2011, Matthias -apoc- Hecker <http://apoc.cc/>
*
- * This file is a collection of javascript and nodejs helper
- * functions. Besides node.js there are no external libraries
- * required.
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/**
+ * Convenient collection of JavaScript and Node.js helper functions,
+ * the methods do not touch any object prototypes. No function has
+ * any internal dependency except the built-in Node.js libraries.
+ *
+ * @fileOverview
*/
-var uri = require('url'),
- fs = require('fs'),
+
+var func = exports;
+
+// built in libraries
+var fs = require('fs'),
+ uri = require('url'),
crypto = require('crypto');
/*****************************************************************************
* Object Functions
*****************************************************************************/
-// merge properties of two objects recursively together
-function object_merge(obj1, obj2) {
+/**
+ * Merge Two Objects Recursively,
+ * converts boolean strings (true, false) to boolean primitives.
+ *
+ * @param object
+ */
+func.object_merge = function(obj1, obj2) {
// @http://stackoverflow.com/questions/171251/how-can-i-merge-properties-
// of-two-javascript-objects-dynamically/383245#383245
for(var key in obj2) {
@@ -41,8 +68,8 @@ function object_merge(obj1, obj2) {
}
}
return obj1;
-}
-exports.object_merge = object_merge;
+};
+// exports.object_merge = object_merge;
exports.object_empty = function(object) {
for(var prop in object) {
Oops, something went wrong.

0 comments on commit da1016a

Please sign in to comment.