Skip to content
Permalink
Browse files

Code inspection clean up

  • Loading branch information...
mitra42 committed May 15, 2019
1 parent 93b09db commit 8c8f3f45e5e3bee61287630e19a54ac9927d79d1
Showing with 44 additions and 62 deletions.
  1. +9 −10 ArchiveItemPatched.js
  2. +3 −4 ArchiveMemberPatched.js
  3. +0 −1 ConfigController.js
  4. +2 −1 CrawlManager.js
  5. +4 −4 INSTALLATION-iiab-raspberry.md
  6. +1 −1 INSTALLATION-rachel.md
  7. +2 −2 INSTALLATION.md
  8. +7 −6 MirrorFS.js
  9. +0 −13 MirrorUI.js
  10. +2 −2 README.md
  11. +1 −1 RELEASENOTES.md
  12. +1 −1 install.sh
  13. +4 −5 internetarchive
  14. +7 −10 mirrorHttp.js
  15. +1 −1 start_ipfs
@@ -19,7 +19,6 @@ const RawBookReaderResponse = require('@internetarchive/dweb-archivecontroller/R
const Util = require('@internetarchive/dweb-archivecontroller/Util');
// Other files from this repo
const MirrorFS = require('./MirrorFS');
const MirrorConfig = require('./MirrorConfig');

// noinspection JSUnresolvedVariable
ArchiveItem.prototype._namepart = function() {
@@ -59,7 +58,7 @@ ArchiveItem.prototype.save = function(opts = {}, cb) {
.members -> <IDENTIFIER>.members.json
.reviews -> <IDENTIFIER>.reviews.json
.files -> <IDENTIFIER>.files.json
{collection_titles, collecton_sort_order, dir, files_count, is_dark, server} -> <IDENTIFIER>.extra.json
{collection_titles, collection_sort_order, dir, files_count, is_dark, server} -> <IDENTIFIER>.extra.json
and .member_cached.json is saved from ArchiveMember not from ArchiveItems
If not already done so, will `fetch_metadata` (but not query, as that may want to be precisely controlled)
@@ -537,15 +536,15 @@ ArchiveItem.prototype.fetch_playlist = function({wantStream=false} = {}, cb) {
wantStream true if want stream) alternative is obj. obj will be processed, stream will always be raw (assuming client processes it)
cb(err, stream|obj) Callback on completion with related items object (can be [])
*/
const itemid = this.itemid; // Its also in this.metadata.identifier but only if done a fetch_metadata
if (itemid) {
const identifier = this.itemid; // Its also in this.metadata.identifier but only if done a fetch_metadata
if (identifier) {
// noinspection JSUnresolvedVariable
const relFilePath = path.join(this._namepart(), this._namepart()+"_playlist.json");
// noinspection JSUnresolvedVariable
MirrorFS.cacheAndOrStream({wantStream, relFilePath,
wantBuff: !wantStream, // Explicit because default for cacheAndOrStream if !wantStream is to return undefined
urls: `https://archive.org/embed/${itemid}?output=json`, // Hard coded, would rather have in Util.gateway.url_playlist but complex
debugname: itemid + "/" + itemid + "_playlist.json"
urls: `https://archive.org/embed/${identifier}?output=json`, // Hard coded, would rather have in Util.gateway.url_playlist but complex
debugname: identifier + "/" + identifier + "_playlist.json"
}, (err, res) => {
// Note that if wantStream, then not doing expansion and saving, but in most cases called will expand with next call.
if (!wantStream && !err) {
@@ -570,15 +569,15 @@ ArchiveItem.prototype.relatedItems = function({wantStream=false, wantMembers=fal
!wantStream && !wantMembers => cb(err, { hits: hit: [ {}* ] }
cb(err, stream|obj) Callback on completion with related items object (can be [])
*/
const itemid = this.itemid; // Its also in this.metadata.identifier but only if done a fetch_metadata
if (itemid) {
const identifier = this.itemid; // Its also in this.metadata.identifier but only if done a fetch_metadata
if (identifier) {
// noinspection JSUnresolvedVariable
const relFilePath = path.join(this._namepart(), this._namepart()+"_related.json");
// noinspection JSUnresolvedVariable
MirrorFS.cacheAndOrStream({wantStream, relFilePath,
wantBuff: !wantStream, // Explicit because default for cacheAndOrStream if !wantStream is to return undefined
urls: Util.gateway.url_related + itemid, //url_related currently ends in /
debugname: itemid + "/" + itemid + "_related.json"
urls: Util.gateway.url_related + identifier, //url_related currently ends in /
debugname: identifier + "/" + identifier + "_related.json"
}, (err, res) => {
// Note that if wantStream, then not doing expansion and saving, but in most cases called will expand with next call.
if (!wantStream && !err) {
@@ -25,7 +25,6 @@ ArchiveMember.read = function({identifier = undefined}, cb) {
identifier: Where to look - can be a real identifier or pseudo-one for a saved search
cb(err, data structure from file)
*/
if (typeof unusedopts === "function") { cb = opts; opts={}; }
const namepart = identifier;
const part = "member";
const relFilePath = path.join(namepart, `${namepart}_${part}.json`);
@@ -48,17 +47,17 @@ ArchiveMember.read = function({identifier = undefined}, cb) {
ArchiveMember.prototype.addCrawlInfo = function({config}, cb) {
Object.assign(this, {crawl: config.crawlInfo(this.identifier)});
cb(null);
}
};
ArchiveMember.addCrawlInfo = function(arr, {config=undefined}={}, cb) { // Should work on an [ArchiveMember*] or [{}*] as same structure
each(arr, (memb, cb2) => memb.addCrawlInfo({config}, cb2), cb);
}
};
ArchiveMember.addCrawlInfoRelated = function(rels, {config=undefined}={}, cb) { // Should work on an [ArchiveMember*] or [{}*] as same structure
const arr = rels.hits.hits;
each(arr, (rel, cb2) => {
Object.assign(rel._source, {crawl: config.crawlInfo(rel._id)});
cb2(null);
}, cb);
}
};
ArchiveMember.prototype.read = function(unusedopts = {}, cb) {
if (typeof unusedopts === "function") { cb = unusedopts; unusedopts={}; }
ArchiveMember.read({identifier: this.identifier}, cb);
@@ -89,7 +89,6 @@ class ConfigController {
// This is subclassed in MirrorConfig to handle specific derivations
}

// noinspection JSUnusedGlobalSymbols
static readYamlSync(filename) {
try {
return yaml.safeLoad(fs.readFileSync(this.resolve(filename), 'utf8'));
@@ -124,7 +124,7 @@ class CrawlManager {
this.cm._taskQ.resume();
}
static empty() {
this.cm._taskQ.remove(o=>true); // Passed {data, priority} but removing all anyway
this.cm._taskQ.remove(task=>true); // Passed {data, priority} but removing all anyway
}
static status() {
return {
@@ -281,6 +281,7 @@ class CrawlItem extends Crawlable {
return true;
}
}

process(cb) {
debug('CrawlManager: processing "%s" %s via %o %o', this.debugname, this.level, this.parent, this.search || "");
this.item = new ArchiveItem({itemid: this.identifier, query: this.query});
@@ -19,7 +19,7 @@ So this is what I did. (Edits welcome, if your experience differed)
* Downloaded Raspbian [Raspbian](https://www.raspberrypi.org/downloads/raspbian/) to your laptop 1GB
* On a Mac:
* downloaded [Etcher](https://www.balena.io/etcher/) (100Mb)
* Run Etcher (its supposed to be able to use the zip, thoguh for this test we used the .img from expanding hte zip), selecting a fresh 16GB SD card as the destination
* Run Etcher (its supposed to be able to use the zip, though for this test we used the .img from expanding hte zip), selecting a fresh 16GB SD card as the destination
* On Windows or Linux, I'm not sure the appropriate steps instead of Etcher.
* Inserted into Raspbian 3, and powered up with Kbd and HDMI and Mouse inserted.
* If at all possible insert Ethernet, otherwise it will work over WiFi with some extra steps.
@@ -28,10 +28,10 @@ So this is what I did. (Edits welcome, if your experience differed)
* Accepted "Next to get started" though I suspect IIAB's comprehensive install gets some of them as well.
* Selected your country, language, keyboard - it shouldnt matter which.
* Changed password since RPis get hacked on default password
* Connected to WiFi (not neccessary if you have Ethernet connected)
* Connected to WiFi (not necessary if you have Ethernet connected)
* It automatically Updated OS - this is big - take a break :-)
* Note that this process failed for me with failures of size and sha, but a restart, after the prompts for password etc,
got me to a partially completed download so didn't have to start from scratch
got me to a partially completed download so I did not have to start from scratch
* You might want to ... Menu/Preferences/Config / Set display to highest resolution

## Install Internet In A Box
@@ -73,7 +73,7 @@ internetarchive_enabled: False

In a browser open: `http://box.lan/admin` id=`iiab-admin` pw=`whatever you set password to during install`

* Note that I've found that `box.lan` doesn't work as documented, and that `box.local` is required instead.
* Note that I've found that `box.lan` does not work as documented, and that `box.local` is required instead.
See [IIAB Issue#1583](https://github.com/iiab/iiab/issues/1583)

Now check dweb-mirror was installed by opening `http://box.local:4244`
@@ -10,7 +10,7 @@ way to give it a full factory reset.

There are docs that come with the Rachel box and are worth reading, however they are inadequate for this task set.

If you have permission to do this then you'll know the passwords so we aren't putting it in this public repo!
If you have permission to do this then you'll know the passwords so we are not putting it in this public repo!

* There are two ways to physically connect the Rachel3+ to the internet either directly via the Internet or via your laptop's Wifi.

@@ -6,7 +6,7 @@ See [README.md] for more info

At the moment this is one set for developing, or use, later I'll split it when its more stable.

Please check for a platform specific INSTALLATION for some platforms. Specifially:
Please check for a platform specific INSTALLATION for some platforms. Specifically:
* [./INSTALLATION-rachel.md] for Rachel
* [./INSTALLATION-iiab-raspberrypi.md] for Internet In A Box (IIAB) on a RaspberryPi
* [./INSTALLATION-raspberrypi.md] for RaspberryPi if you don't want Internet In A Box
@@ -63,7 +63,7 @@ sudo yarn install
# If it fails, then running it again is safe.
```
The example above would install dweb-mirror as `/usr/local/node_modules/@internetarchive/dweb-mirror`
which is refered to as `<wherever>/dweb-mirror` in the rest of this INSTALLATION
which is referred to as `<wherever>/dweb-mirror` in the rest of this INSTALLATION

Now skip to step 4

@@ -11,6 +11,7 @@ const detect = require('async/detect');
const detectSeries = require('async/detectSeries');
const each = require('async/each');
const waterfall = require('async/waterfall');
// noinspection ES6ConvertVarToLetConst
var exec = require('child_process').exec;

// other packages of ours
@@ -24,7 +25,7 @@ const HashStore = require('./HashStore');
function multihash58sha1(buf) { return multihashes.toB58String(multihashes.encode(buf, 'sha1')); }


class MirrorFS { //TODO-API needs uodating
class MirrorFS { //TODO-API needs updating
/*
Utility subclass that knows about the file system.
@@ -228,9 +229,9 @@ class MirrorFS { //TODO-API needs uodating
cb(err, filepath) - Careful, its err,undefined if not found unlike checkWhereValidFile
*/
const scales = [];
for(let i=Math.floor(scale); i>0; i--) { scales.push(i); }; // A = e.g. [ 8...1 ]
for(let i=Math.floor(scale); i>0; i--) { scales.push(i); } // A = e.g. [ 8...1 ]
detectSeries(scales.map(s => `${relFileDir}/scale${s}/rotate${rotate}/${file}`),
(rel, cb2) => this.checkWhereValidFile(rel, {}, (err, res) => cb2(null, !err)), // Find the first place having a file bigger or same size as
(rel, cb2) => this.checkWhereValidFile(rel, {}, (err, unusedRes) => cb2(null, !err)), // Find the first place having a file bigger or same size as
cb
)
}
@@ -422,7 +423,7 @@ class MirrorFS { //TODO-API needs uodating
debug(`Closed ${debugname} size=${writable.bytesWritten}`);
this.seed({relFilePath, directory: this._copyDirectory()}, (err, res) => { }); // Seed to IPFS, WebTorrent etc
//Ignore err & res, its ok to fail to seed and will be logged inside seed()
// Also - its running background, we arent making caller wait for it to complete
// Also - its running background, we are not making caller wait for it to complete
// noinspection JSUnresolvedVariable
if (!wantStream) { // If wantStream then already called cb, otherwise cb signifies file is written
callbackEmptyOrData(newFilePath);
@@ -490,7 +491,7 @@ class MirrorFS { //TODO-API needs uodating
const tablename = `${algorithm}.relfilepath`;
each( (cacheDirectories && cacheDirectories.length) ? cacheDirectories : this.directories,
(cacheDirectory, cb1) => {
this.hashstores[cacheDirectory].destroy(tablename, (err, res)=> {
this.hashstores[cacheDirectory].destroy(tablename, (err, unusedRes)=> {
if (err) {
debug("Unable to destroy hashstore %s in %s", tablename,cacheDirectory);
cb1(err);
@@ -514,7 +515,7 @@ class MirrorFS { //TODO-API needs uodating
debug("loadHashTable saw error: %s", err.message);
cb2(err);
} else {
this.hashstores[cacheDirectory].put(tablename, multiHash, relFilePath, (err, res) => {
this.hashstores[cacheDirectory].put(tablename, multiHash, relFilePath, (err, unusedRes) => {
if (err) { debug("failed to put table:%s key:%s val:%s %s", tablename, multiHash, relFilePath, err.message); cb2(err); } else {
cb2(null, relFilePath)
}

This file was deleted.

@@ -38,7 +38,7 @@ browse to [http://localhost:4244] which will open the UI in the browser and it s

If you are remote from the machine, then browser to: `http://<IP of machine>:4244`

On IIAB the server can be accessed at [http://box.lan:4244] (try [http://box.local:4244] if that doesn't work)
On IIAB the server can be accessed at [http://box.lan:4244] (try [http://box.local:4244] if that does not work)
If you don’t get a Archive UI then look at the server log (in browser console)
to see for any “FAILING” log lines which indicate a problem.

@@ -139,7 +139,7 @@ As the project develops, this file will be editable via a UI.
Dweb-Mirror is installable on most Linux based systems. Specific installation instructions are available for:
[RACHEL 3+ from World Possible](INSTALLATION-rachel.md); [Internet In A Box on Raspberry Pi](INSTALLATION-iiab-raspberry.md); and [directly on Raspberry PI](INSTALLATION-raspberrypi.md).

For anything else (incuding MacOSX), please follow [generic instructions](INSTALLATION.md) and please let us know how it went.
For anything else (including MacOSX), please follow [generic instructions](INSTALLATION.md) and please let us know how it went.

## Update

@@ -37,7 +37,7 @@ This release integrates IPFS integration, so any files in the crawled cache are
See [#11](https://github.com/internetarchive/dweb-mirror/issues/11) for current state.

* 0.2.19: Installer for bookreader
* 0.2.18: Iniitalize user config file if not there
* 0.2.18: Initialize user config file if not there
* 0.2.16: Improved behavior standalone or when cant see gateway, installation documentation
* 0.2.15: bookreader bug when not cached and bug in installer
* 0.2.14: Add configuration for IIAB (partial)
@@ -51,4 +51,4 @@ lnfirst . ../../../bookreader/BookReader \
popd
popd
#ls -al ${ARCHIVEUI}
./install_ipfs.sh # IPFS install in a seperate script as its prone to failure
./install_ipfs.sh # IPFS install in a separate script as its prone to failure
@@ -46,7 +46,7 @@ const opts = getopts(process.argv.slice(2),{
//default: {transport: "HTTP"}, // Getting defaults from yaml via MirrorConfig
"unknown": option => { if (!optsInt.includes(option)) { console.log("Unknown option", option, ", 'internetarchive -h' for help"); process.exit()} }
});
const httpOrHttps = "http"; // This server is running on http, not https (at least currenty)
const httpOrHttps = "http"; // This server is running on http, not https (at least currently)

//TODO allow specifying "crawl item/filename" to get just one file see https://github.com/internetarchive/dweb-mirror/issues/108

@@ -170,9 +170,10 @@ function optsToConfig() {
return new Error("ERROR: Cannot specify search with rows or depth arguments");
}
if (opts.debugidentifier.length) {
var debugidentifier = opts.debugidentifier };
// noinspection ES6ConvertVarToLetConst
var debugidentifier = opts.debugidentifier }
if (opts.verbose || opts.dummy) {
debug( "Config:"); debug(yaml.safeDump(Object.filter(config, (key, value) => key !== "configOpts")));
debug( "Config:"); debug(yaml.safeDump(Object.filter(config, (key, unusedValue) => key !== "configOpts")));
}
return null;
}
@@ -230,8 +231,6 @@ function connect(cb) {
}
if (!opts.dummy) {
DwebTransports.connect(connectOpts, unusedErr => {
//TODO-MIRROR this is working around default that HTTP doesnt officially support streams, till sure can use same interface with http & WT
const Thttp = DwebTransports.http();
cb(unusedErr);
});
}

0 comments on commit 8c8f3f4

Please sign in to comment.
You can’t perform that action at this time.