hyperdrive 8

.gitignore

@@ -1,5 +1,3 @@
 node_modules
-*.mp4
+sandbox.js
 sandbox
-layout.txt
-*.db

.travis.yml

@@ -1,6 +1,4 @@
 language: node_js
 node_js:
-  - '0.10'
-  - '0.12'
-  - '4.0'
-  - '6.0'
+  - "6"
+  - "4"

README.md

@@ -1,271 +1,156 @@
-# hyperdrive
+# Hyperdrive
 
-A file sharing network based on [rabin](https://github.com/maxogden/rabin) file chunking and [append only feeds of data verified by merkle trees](https://github.com/mafintosh/hypercore).
+Hyperdrive is a secure, real time distributed file system
 
-```
-npm install hyperdrive
-```
+This readme describes version 8 is coming out soon, try the preview here
 
-[![build status](http://img.shields.io/travis/mafintosh/hyperdrive.svg?style=flat)](http://travis-ci.org/mafintosh/hyperdrive)
-
-If you are interested in learning how hyperdrive works on a technical level a specification is available in the [Dat docs repo](https://github.com/datproject/docs/blob/master/docs/hyperdrive_spec.md)
+``` js
+npm install mafintosh/hyperdrive
+```
 
 ## Usage
 
-First create a new feed and share it
+Hyperdrive aims to implement the same API as Node.js' core fs module.
 
 ``` js
 var hyperdrive = require('hyperdrive')
-var level = require('level')
-var swarm = require('discovery-swarm')()
-
-var db = level('./hyperdrive.db')
-var drive = hyperdrive(db)
-
-var archive = drive.createArchive()
-var ws = archive.createFileWriteStream('hello.txt') // add hello.txt
-
-ws.write('hello')
-ws.write('world')
-ws.end()
+var archive = hyperdrive('./my-first-hyperdrive') // content will be stored in this folder
 
-var link = archive.key.toString('hex')
-console.log(link, '<-- this is your hyperdrive link')
-
-// the archive is now ready for sharing.
-// we can use swarm to replicate it to other peers
-swarm.listen()
-swarm.join(new Buffer(link, 'hex'))
-swarm.on('connection', function (connection) {
-  connection.pipe(archive.replicate()).pipe(connection)
-})
-```
-
-Then we can access the content from another process with the following code
-
-``` js
-var swarm = require('discovery-swarm')()
-var hyperdrive = require('hyperdrive')
-var level = require('level')
-
-var db = level('./another-hyperdrive.db')
-var drive = hyperdrive(db)
-
-var link = new Buffer('your-hyperdrive-link-from-the-above-example', 'hex')
-var archive = drive.createArchive(link)
-
-swarm.listen()
-swarm.join(link)
-swarm.on('connection', function (connection) {
-  connection.pipe(archive.replicate()).pipe(connection)
-  archive.get(0, function (err, entry) { // get the first file entry
-    console.log(entry) // prints {name: 'hello.txt', ...}
-    var stream = archive.createFileReadStream(entry)
-    stream.on('data', function (data) {
-      console.log(data) // <-- file data
-    })
-    stream.on('end', function () {
-      console.log('no more data')
+archive.writeFile('/hello.txt', 'world', function (err) {
+  if (err) throw err
+  archive.readdir('/', function (err, list) {
+    if (err) throw err
+    console.log(list) // prints ['hello.txt']
+    archive.readFile('/hello.txt', 'utf-8', function (err, data) {
+      if (err) throw err
+      console.log(data) // prints 'world'
     })
   })
 })
 ```
 
-If you want to write/read files to the file system provide a storage driver as the file option
-
-``` js
-var raf = require('random-access-file') // a storage driver that writes to the file system
-var archive = drive.createArchive({
-  file: function (name) {
-    return raf('my-download-folder/' + name)
-  }
-})
-```
-
-## API
-
-#### `var drive = hyperdrive(db)`
-
-Create a new hyperdrive instance. db should be a [levelup](https://github.com/level/levelup) instance.
-
-#### `var archive = drive.createArchive([key], [options])`
-
-Creates an archive instance. If you want to download/upload an existing archive provide the archive key
-as the first argument. Options include
+A big difference is that you can replicate the file system to other computers! All you need is a stream.
 
 ``` js
-{
-  live: false, // set this to share the archive without finalizing it
-  sparse: false, // set this to only download the pieces of the feed you are requesting / prioritizing
-  file: function (name) {
-    // set this to determine how file data is stored.
-    // the storage instance should implement the hypercore storage api
-    // https://github.com/mafintosh/hypercore#storage-api
-    return someStorageInstance
-  }
-}
-```
-
-If you do not provide the file option all file data is stored in the leveldb.
+var net = require('net')
 
-If the `metadata` and `content` hypercore feeds were already created, they can be passed in directly as options:
+// ... on one machine
 
-```js
-var archive = drive.createArchive(key, {
-  metadata: core.createFeed(key),
-  content: core.createFeed(contentKey)
+var server = net.createServer(function (socket) {
+  socket.pipe(archive.replicate()).pipe(socket)
 })
-```
-
-#### `archive.key`
-
-A buffer that verifies the archive content. In live mode this is a 32 byte public key.
-Otherwise it is a 32 byte hash.
 
-#### `archive.live`
+server.listen(10000)
 
-Boolean whether archive is live. `true` by default. Note that its only populated after archive.open(cb) has been fired.
+// ... on another
 
-#### `archive.append(entry, callback)`
+var clonedArchive = hyperdrive('./my-cloned-hyperdrive')
+var socket = net.connect(10000)
 
-Append an entry to the archive. Only possible if this is an live archive you originally created
-or an unfinalized archive.
+socket.pipe(clonedArchive.replicate()).pipe(socket)
+```
 
-If you set the file option in the archive constructor you can use this method to append an already
-existing file to the archive.
+It also comes with build in versioning and real time replication. See more below.
 
-``` js
-var archive = drive.createArchive({
-  file: function (name) {
-    console.log('returning storage for', name)
-    return raf(name)
-  }
-})
+## API
 
-archive.append('hello.txt', function () {
-  console.log('hello.txt was read and appended')
-})
-```
+#### `var archive = hyperdrive(storage, [key], [options])`
 
-#### `archive.finalize([callback])`
+Create a new hyperdrive. Storage should be a function or a string.
 
-Finalize the archive. You need to do this before sharing it if the archive is not live (it is live per default).
+If storage is a string content will be stored inside that folder.
 
-You should only call `archive.finalize()` on archives you own.
+If storage is a function it is called with a string name for each abstract-random-access instance that is needed
+to store the archive.
 
-#### `archive.get(index, [options], callback)`
+#### `var stream = archive.replicate([options])`
 
-Reads an entry from the archive. Options include:
+Replicate this archive. Options include
 
 ``` js
 {
-  timeout: 1000 // time out after 1000ms. Default is Infinity
+  live: false // keep replicating
 }
 ```
 
-#### `archive.download(index, callback)`
+#### `archive.version`
+
+Get the current version of the archive (incrementing number).
 
-Fully downloads a file / entry from the archive and calls the callback afterwards.
+#### `archive.key`
+
+The public key identifying the archive.
 
-#### `archive.close([callback])`
+#### `archive.discoveryKey`
 
-Closes and releases all resources used by the archive. Call this when you are done using it.
+A key derived from the public key that can be used to discovery other peers sharing this archive.
 
-#### `archive.on('download', data)`
+#### `archive.on('ready')`
 
-Emitted every time a piece of data is downloaded
+Emitted when the archive is fully ready and all properties has been populated.
 
-#### `archive.on('upload', data)`
+#### `archive.on('error', err)`
 
-Emitted every time a piece of data is uploaded
+Emitted when a critical error during load happened.
 
-#### `var rs = archive.list(opts={}, cb)`
+#### `var oldDrive = archive.checkout(version)`
 
-Returns a readable stream of all entries in the archive.
+Checkout a readonly copy of the archive at an old version.
 
-* `opts.offset` - start streaming from this offset (default: 0)
-* `opts.limit` - stop streaming at this offset (default: no limit)
-* `opts.live` - keep the stream open as new updates arrive (default: `true` if no callback given, `false` if callback is given)
+#### `var stream = archive.history([options])`
 
-You can collect the results of the stream with `cb(err, entries)`.
+Get a stream of all changes and their versions from this archive.
 
-#### `var rs = archive.createFileReadStream(entry, [options])`
+#### `var stream = archive.createReadStream(name, [options])`
 
-Returns a readable stream of the file content of an file in the archive.
+Read a file out as a stream. Similar to fs.createReadStream.
 
 Options include:
 
 ``` js
 {
-  start: startOffset, // defaults to 0
-  end: endOffset // defaults to file.length
+  start: optionalByteOffset, // similar to fs
+  end: optionalInclusiveByteEndOffset, // similar to fs
+  length: optionalByteLength
 }
 ```
 
-#### `var ws = archive.createFileWriteStream(entry)`
+#### `archive.readFile(name, [encoding], callback)`
 
-Returns a writable stream that writes a new file to the archive. Only possible if the archive is live and you own it
-or if the archive is not finalized.
+Read an entire file into memory. Similar to fs.readFile.
 
-#### `var cursor = archive.createByteCursor(entry, [options])`
+#### `var stream = archive.createWriteStream(name, [options])`
 
-Creates a cursor that can seek and traverse parts of the file.
+Write a file as a stream. Similar to fs.createWriteStream.
 
-``` js
-var cursor = archive.createByteCursor('hello.txt')
+#### `archive.writeFile(name, buffer, [options], [callback])`
 
-// seek to byte offset 10000 and read the rest.
-cursor.seek(10000, function (err) {
-  if (err) throw err
-  cursor.next(function loop (err, data) {
-    if (err) throw err
-    if (!data) return console.log('no more data')
-    console.log('cursor.position is ' + cursor.position)
-    console.log('read', data.length, 'bytes')
-    cursor.next(loop)
-  })
-})
-```
+Write a file from a single buffer. Similar to fs.writeFile.
 
-Options include
+#### `archive.unlink(name, [callback])`
 
-``` js
-{
-  start: startOffset, // defaults to 0
-  end: endOffset // defaults to file.length
-}
-```
+Unlinks (deletes) a file. Similar to fs.unlink.
 
-#### `var stream = archive.replicate([options])`
+#### `archive.mkdir(name, [options], [callback])`
 
-Pipe this stream together with another peer that is interested in the same archive to replicate the content.
-Options include:
+Explictly create an directory. Similar to fs.mkdir
 
-``` js
-{
-  upload: true, // upload content to remote peer
-  download: true // downlod content from remote peer
-}
-```
+#### `archive.rmdir(name, [callback])`
 
-#### `archive.unreplicate([stream])`
+Delete an empty directory. Similar to fs.rmdir.
 
-Stop replicating an archive to all streams, or to a specific stream.
+#### `archive.readdir(name, [callback])`
 
-#### `archive.countDownloadedBlocks(entry)`
+Lists a directory. Similar to fs.readdir.
 
-Count the number of blocks in the entry that have been downloaded.
-You can calculate the file's download progress, as a percentage, with:
+#### `archive.stat(name, callback)`
 
-```js
-var downloaded = archive.countDownloadedBlocks(entry)
-var progress =  downloaded / entry.blocks
-```
+Stat an entry. Similar to fs.stat.
 
-#### `archive.isEntryDownloaded(entry)`
+#### `archive.lstat(name, callback)`
 
-Has all of the entry's blocks been downloaded?
+Stat an entry but do not follow symlinks. Similar to fs.lstat.
 
-## License
+#### `archive.access(name, callback)`
 
-MIT
+Similar to fs.access.