first tests are passing

package.json

@@ -37,6 +37,7 @@
     "aegir": "^10.0.0",
     "buffer-loader": "^0.0.1",
     "chai": "^3.5.0",
+    "datastore-fs": "^0.1.0",
     "interface-pull-blob-store": "~0.6.0",
     "leveldown": "^1.6.0",
     "lodash": "^4.17.4",
@@ -46,7 +47,10 @@
   },
   "dependencies": {
     "async": "^2.1.4",
+    "datastore-core": "^0.1.0",
+    "datastore-level": "^0.1.0",
     "debug": "^2.6.2",
+    "interface-datastore": "^0.1.1",
     "ipfs-block": "~0.5.5",
     "lock": "^0.1.3",
     "lock-me": "^1.0.2",

src/index.js

@@ -13,12 +13,15 @@ const Multiaddr = require('multiaddr')
 const Buffer = require('safe-buffer').Buffer
 const assert = require('assert')
 const path = require('path')
+const debug = require('debug')
 
 const version = require('./version')
 const config = require('./config')
 const blockstore = require('./blockstore')
 const lock = require('./lock')
 
+const log = debug('repo')
+
 const apiFile = new Key('api')
 const flatfsDirectory = 'blocks'
 const levelDirectory = 'datastore'
@@ -52,29 +55,36 @@ class IpfsRepo {
     if (!this.closed) {
       return setImmediate(callback)
     }
+    log('opening at %s', this.path)
 
     const FsStore = this.options.fs
-    this.fsStore = new FsStore(this.path)
+    this.fsStore = new FsStore(this.path, {extension: ''})
 
     this.version = version(this.fsStore)
     this.config = config(this.fsStore)
 
     // check if the repo is already initialized
     waterfall([
       (cb) => this._isInitialized(cb),
-      (cb) => lock.lock(cb),
+      (cb) => lock.lock(this.path, cb),
       (lck, cb) => {
+        log('aquired repo.lock')
         this.lockfile = lck
         this.version.check(cb)
       },
-      (cb) => ShardingStore.createOrOpen(new FsStore(path.join(this.path, flatfsDirectory)), cb),
-
+      (cb) => {
+        log('creating flatfs')
+        const s = new FsStore(path.join(this.path, flatfsDirectory))
+        const shard = new core.shard.NextToLast(2)
+        ShardingStore.createOrOpen(s, shard, cb)
+      },
       (flatfs, cb) => {
-        const level = new LevelStore(path.join(this.path, levelDirectory), {db: this.options.level})
-
+        log('Flatfs store opened')
         this.store = new MountStore([{
           prefix: new Key('/'),
-          datstore: level
+          datastore: new LevelStore(path.join(this.path, levelDirectory), {
+            db: this.options.level
+          })
         }, {
           prefix: new Key(flatfsDirectory),
           datastore: flatfs
@@ -86,8 +96,12 @@ class IpfsRepo {
       }
     ], (err) => {
       if (err && this.lockfile) {
-        return this.lockfile.close(() => callback(err))
+        return this.lockfile.close((err2) => {
+          log('error removing lock', err2)
+          callback(err)
+        })
       }
+
       callback(err)
     })
   }
@@ -123,8 +137,14 @@ class IpfsRepo {
       return callback(new Error('repo is already closed'))
     }
 
+    log('closing at: %s', this.path)
     series([
-      (cb) => this.fsStore.delete(apiFile, cb),
+      (cb) => this.fsStore.delete(apiFile, (err) => {
+        if (err && err.message.startsWith('ENOENT')) {
+          return cb()
+        }
+        cb(err)
+      }),
       (cb) => this.store.close(cb),
       (cb) => this.fsStore.close(cb),
       (cb) => {

src/lock.js

@@ -1,13 +1,14 @@
 'use strict'
 
-const lockMe = require('lock-me')
+const Lock = require('lock-me')
 const path = require('path')
 const debug = require('debug')
 const fs = require('fs')
 
 const log = debug('repo:lock')
 
 const lockFile = 'repo.lock'
+const lock = new Lock()
 
 /**
  * Lock the repo in the given dir.
@@ -19,7 +20,7 @@ const lockFile = 'repo.lock'
 exports.lock = (dir, callback) => {
   const file = path.join(dir, lockFile)
   log('locking %s', file)
-  lockMe.lock(file, callback)
+  lock(file, callback)
 }
 
 /**
@@ -37,7 +38,7 @@ exports.locked = (dir, callback) => {
     log('file does not exist: %s', file)
   }
 
-  lockMe.lock(file, (err, lck) => {
+  lock(file, (err, lck) => {
     if (err) {
       log('already locked: %s', err.message)
       return callback(null, true)

src/version.js

@@ -1,6 +1,8 @@
 'use strict'
 
 const Key = require('interface-datastore').Key
+const debug = require('debug')
+const log = debug('repo:version')
 
 const versionKey = new Key('version')
 const repoVersion = 5
@@ -46,10 +48,12 @@ module.exports = (store) => {
      * @returns {void}
      */
     check (callback) {
+      log('checking version')
       this.get((err, version) => {
         if (err) {
           return callback(err)
         }
+        log('comparing version: %s and %s', version, repoVersion)
         if (version !== repoVersion) {
           return callback(new Error(`version mismatch: expected v${repoVersion}, found v${version}`))
         }

test/node.js

@@ -2,10 +2,10 @@
 
 'use strict'
 
-const expect = require('chai').expect
 const ncp = require('ncp').ncp
 const rimraf = require('rimraf')
 const path = require('path')
+const series = require('async/series')
 
 const IPFSRepo = require('../src')
 
@@ -14,29 +14,24 @@ describe('IPFS Repo Tests on on Node.js', () => {
   const date = Date.now().toString()
   const repoPath = testRepoPath + '-for-' + date
 
-  let repo
+  const repo = new IPFSRepo(repoPath, {
+    fs: require('datastore-fs'),
+    level: require('leveldown')
+  })
 
   before((done) => {
-    console.log('repoPath: %s', repoPath)
-    repo = new IPFSRepo(repoPath, {
-      fs: require('datastore-fs'),
-      level: require('leveldown')
-    })
-    ncp(testRepoPath, repoPath, (err) => {
-      expect(err).to.not.exist
-
-      repo.open((err) => {
-        expect(err).to.not.exist
-        done()
-      })
-    })
+    series([
+      (cb) => ncp(testRepoPath, repoPath, cb),
+      (cb) => repo.open(cb)
+    ], done)
   })
 
   after((done) => {
-    rimraf(repoPath, (err) => {
-      expect(err).to.not.exist
-      done()
-    })
+    series([
+      (cb) => repo.close(cb),
+      (cb) => rimraf(repoPath, cb)
+    ], done)
   })
+
   require('./repo-test')(repo)
 })

test/repo-test.js

@@ -61,18 +61,17 @@ module.exports = (repo) => {
       it('get version', (done) => {
         repo.version.get((err, version) => {
           expect(err).to.not.exist
-          expect(version).to.be.a('string')
-          expect(Number(version)).to.be.a('number')
+          expect(version).to.be.eql(5)
           done()
         })
       })
 
       it('set version', (done) => {
-        repo.version.set('9000', (err) => {
+        repo.version.set(9000, (err) => {
           expect(err).to.not.exist
           repo.version.get((err, version) => {
             expect(err).to.not.exist
-            expect(version).to.equal('9000')
+            expect(version).to.equal(9000)
             done()
           })
         })

test/test-repo/blocks/SHARDING

@@ -0,0 +1 @@
+/repo/flatfs/shard/v1/next-to-last/2

test/test-repo/blocks/_README

@@ -0,0 +1,30 @@
+This is a repository of IPLD objects. Each IPLD object is in a single file,
+named <base32 encoding of cid>.data. Where <base32 encoding of cid> is the
+"base32" encoding of the CID (as specified in
+https://github.com/multiformats/multibase) without the 'B' prefix.
+All the object files are placed in a tree of directories, based on a
+function of the CID. This is a form of sharding similar to
+the objects directory in git repositories. Previously, we used
+prefixes, we now use the next-to-last two charters.
+
+    func NextToLast(base32cid string) {
+      nextToLastLen := 2
+      offset := len(base32cid) - nextToLastLen - 1
+      return str[offset : offset+nextToLastLen]
+    }
+
+For example, an object with a base58 CIDv1 of
+
+    zb2rhYSxw4ZjuzgCnWSt19Q94ERaeFhu9uSqRgjSdx9bsgM6f
+
+has a base32 CIDv1 of
+
+    BAFKREIA22FLID5AJ2KU7URG47MDLROZIH6YF2KALU2PWEFPVI37YLKRSCA
+
+and will be placed at
+
+    SC/AFKREIA22FLID5AJ2KU7URG47MDLROZIH6YF2KALU2PWEFPVI37YLKRSCA.data
+
+with 'SC' being the last-to-next two characters and the 'B' at the
+beginning of the CIDv1 string is the multibase prefix that is not
+stored in the filename.

test/test-repo/version

@@ -1 +1 @@
-4
+5