Hotfix/disk content type

hub/package.json

@@ -1,6 +1,6 @@
 {
   "name": "gaia-hub",
-  "version": "2.2.1",
+  "version": "2.2.2",
   "description": "",
   "main": "index.js",
   "engines": {

hub/src/server/drivers/diskDriver.js

@@ -1,6 +1,6 @@
 /* @flow */
 import fs from 'fs'
-import { BadPathError } from '../errors'
+import { BadPathError, InvalidInputError } from '../errors'
 import logger from 'winston'
 import Path from 'path'
 
@@ -11,6 +11,8 @@ type DISK_CONFIG_TYPE = { diskSettings: { storageRootDirectory?: string },
                           pageSize?: number,
                           readURL?: string }
 
+const METADATA_DIRNAME = '.gaia-metadata'
+
 class DiskDriver implements DriverModel {
   storageRootDirectory: string
   readURL: string
@@ -135,11 +137,17 @@ class DiskDriver implements DriverModel {
     .then(() => {
       const path = args.path
       const topLevelDir = args.storageTopLevel
+      const contentType = args.contentType
 
       if (!topLevelDir) {
         throw new BadPathError('Invalid Path')
       }
 
+      if (contentType.length > 255) {
+        // no way this is valid 
+        throw new InvalidInputError('Invalid content-type')
+      }
+
       const abspath = Path.join(this.storageRootDirectory, topLevelDir, path)
       if (!DiskDriver.isPathValid(abspath)) {
         throw new BadPathError('Invalid Path')
@@ -163,6 +171,17 @@ class DiskDriver implements DriverModel {
       const writePipe = fs.createWriteStream(abspath, { mode: 0o600, flags: 'w' })
       args.stream.pipe(writePipe)
 
+      // remember content type in $storageRootDir/.gaia-metadata/$address/$path
+      // (i.e. these files are outside the address bucket, and are thus hidden)
+      const contentTypePath = Path.join(
+        this.storageRootDirectory, METADATA_DIRNAME, topLevelDir, path)
+
+      const contentTypeDirPath = Path.dirname(contentTypePath)
+      this.mkdirs(contentTypeDirPath)
+
+      fs.writeFileSync(
+        contentTypePath, JSON.stringify({ 'content-type': contentType }), { mode: 0o600 })
+
       return `${this.readURL}${Path.join(topLevelDir, path)}`
     })
   }

hub/src/server/errors.js

@@ -21,4 +21,10 @@ export class NotEnoughProofError {
   }
 }
 
-
+export class InvalidInputError {
+  constructor (message) {
+    this.name = 'InvalidInputError'
+    this.message = message
+    this.stack = (new Error(message)).stack
+  }
+}

hub/test/test.js

@@ -6,6 +6,7 @@ const FetchMock = require('fetch-mock')
 let request = require('supertest')
 let bitcoin = require('bitcoinjs-lib')
 let fs = require('fs')
+let Path = require('path')
 
 const { Readable, Writable } = require('stream');
 
@@ -534,9 +535,10 @@ function testDiskDriver() {
   const DiskDriver = require(diskDriverImport)
 
   test('diskDriver', (t) => {
-    t.plan(4)
+    t.plan(5)
     const driver = new DiskDriver(config)
     const prefix = driver.getReadURLPrefix()
+    const storageDir = driver.storageRootDirectory
     const s = new Readable()
     s._read = function noop() {}
     s.push('hello world')
@@ -553,7 +555,11 @@ function testDiskDriver() {
           contentType: 'application/octet-stream',
           contentLength: 12 }))
       .then((readUrl) => {
+        const filePath = Path.join(storageDir, '12345', 'foo/bar.txt')
+        const metadataPath = Path.join(storageDir, '.gaia-metadata', '12345', 'foo/bar.txt')
         t.ok(readUrl.startsWith(prefix + '12345'), `${readUrl} must start with readUrlPrefix ${prefix}12345`)
+        t.equal(JSON.parse(fs.readFileSync(metadataPath).toString())['content-type'], 'application/octet-stream',
+          'Content-type metadata was written')
       })
       .then(() => driver.listFiles('12345'))
       .then((files) => {

package.json

@@ -4,6 +4,6 @@
     "node": "8.1.x"
   },
   "scripts": {
-    "postinstall": "NODE_ENV=development npm install --prefix hub"
+    "postinstall": "NODE_ENV=development npm install --prefix hub && NODE_ENV=development npm install --prefix reader"
   }
 }

reader/.babelrc

@@ -0,0 +1,3 @@
+{
+  "presets": ["env", "flow"]
+}

reader/.eslintrc

@@ -0,0 +1,33 @@
+{
+  "parser": "babel-eslint",
+  "plugins": ["flowtype"],
+  "env": {
+    "browser": false,
+    "es6": true,
+    "node": true,
+  },
+  "rules": {
+    "comma-dangle": ["error", "never"],
+    "quotes": [2, "single"],
+    "eol-last": 2,
+    "no-debugger": 1,
+    "no-mixed-requires": 0,
+    "no-underscore-dangle": 0,
+    "no-multi-spaces": 0,
+    "no-trailing-spaces": 0,
+    "no-extra-boolean-cast": 0,
+    "no-undef": 2,
+    "no-unused-vars": 2,
+    "no-var": 2,
+    "no-param-reassign": 0,
+    "no-else-return": 0,
+    "no-console": 0,
+    "prefer-const": 2,
+    "new-cap": 0,
+    "camelcase": 2,
+    "brace-style": 2,
+    "semi": [2, "never"],
+    "flowtype/define-flow-type": 1,
+    "valid-jsdoc": ["error"]
+  }
+}

reader/.flowconfig

@@ -0,0 +1,12 @@
+[ignore]
+.*/node_modules/documentation/*
+.*/node_modules/.staging/*
+.*/node_modules/npm/*
+
+[include]
+
+[libs]
+
+[options]
+
+[lints]

reader/package.json

@@ -0,0 +1,51 @@
+{
+  "name": "gaia-reader",
+  "version": "2.2.2",
+  "description": "",
+  "main": "index.js",
+  "engines": {
+    "node": "^8"
+  },
+  "dependencies": {
+    "body-parser": "^1.18.1",
+    "cors": "^2.8.4",
+    "express": "^4.15.4",
+    "express-winston": "^2.4.0",
+    "node-fetch": "^2.0.0",
+    "winston": "^2.3.1"
+  },
+  "devDependencies": {
+    "babel-cli": "^6.26.0",
+    "babel-eslint": "^8.2.2",
+    "babel-preset-env": "^1.6.1",
+    "babel-preset-flow": "^6.23.0",
+    "eslint": "^4.18.1",
+    "eslint-plugin-flowtype": "^2.46.3",
+    "fetch-mock": "^6.4.2",
+    "flow-bin": "^0.71.0",
+    "nock": "^9.1.9",
+    "nyc": "^13.0.0",
+    "proxyquire": "^2.0.1",
+    "supertest": "^3.0.0",
+    "tape": "^4.9.0"
+  },
+  "bin": {
+    "blockstack-gaia-reader": "./lib/index.js"
+  },
+  "scripts": {
+    "start": "npm run build && node lib/index.js",
+    "build": "npm run lint && babel src -d lib && chmod +x lib/index.js",
+    "flow": "flow",
+    "lint": "eslint src",
+    "test-inner": "npm run build && npm run flow && node ./test/test.js",
+    "test": "nyc --reporter=text npm run test-inner"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/blockstack/gaia.git"
+  },
+  "authors": [
+    "Aaron Blankstein (aaron@blockstack.com)",
+    "Jude Nelson (jude@blockstack.com)"
+  ]
+}

reader/src/config.js

@@ -0,0 +1,37 @@
+import winston from 'winston'
+import fs from 'fs'
+import process from 'process'
+
+const configDefaults = {
+  argsTransport: {
+    level: 'debug',
+    handleExceptions: true,
+    timestamp: true,
+    stringify: true,
+    colorize: true,
+    json: true
+  },
+  regtest: false,
+  testnet: false,
+  port: 8008,
+  diskSettings: {
+    storageRootDirectory: '/tmp/gaia-disk'
+  }
+}
+
+export function getConfig() {
+  const configPath = process.env.CONFIG_PATH || process.argv[2] || './config.json'
+  let config
+  try {
+    config = Object.assign(
+      {}, configDefaults, JSON.parse(fs.readFileSync(configPath).toString()))
+  } catch (e) {
+    config = Object.assign({}, configDefaults)
+  }
+
+  winston.configure({ transports: [
+    new winston.transports.Console(config.argsTransport) ] })
+
+  return config
+}
+

reader/src/http.js

@@ -0,0 +1,58 @@
+/* @flow */
+
+import express from 'express'
+import expressWinston from 'express-winston'
+import logger from 'winston'
+import cors from 'cors'
+import Path from 'path'
+
+import {
+  GaiaDiskReader
+} from './server'
+
+export function makeHttpServer(config: Object) {
+  const app = express()
+  const server = new GaiaDiskReader(config)
+
+  app.config = config
+
+  app.use(expressWinston.logger({
+    transports: logger.loggers.default.transports }))
+
+  app.use(cors())
+
+  app.get(/\/([a-zA-Z0-9]+)\/(.+)/, (req: express.request, res: express.response) => {
+    let filename = req.params[1]
+    if (filename.endsWith('/')) {
+      filename = filename.substring(0, filename.length - 1)
+    }
+    const address = req.params[0]
+
+    return server.handleGet(address, filename)
+      .then((fileInfo) => {
+        const exists = fileInfo.exists
+        const contentType = fileInfo.contentType
+
+        if (!exists) {
+          return res.status(404).send('File not found')
+        }
+
+        const opts = {
+          root: config.storageRootDirectory,
+          headers: {
+            'content-type': contentType
+          }
+        }
+        const path = Path.join(address, filename)
+
+        return res.sendFile(path, opts)
+      })
+      .catch((err) => {
+        logger.error(err)
+        return res.status(400).send('Could not return file')
+      })
+  })
+
+  return app
+}
+

reader/src/index.js

@@ -0,0 +1,12 @@
+#!/usr/bin/env node
+import winston from 'winston'
+import { makeHttpServer } from './http.js'
+import { getConfig } from './config.js'
+
+const conf = getConfig()
+const app = makeHttpServer(conf)
+
+app.listen(
+  app.config.port,
+  () => winston.warn(`server starting on port ${app.config.port} in ${app.settings.env} mode`))
+

reader/src/server.js

@@ -0,0 +1,42 @@
+/* @flow */
+
+import Path from 'path'
+import fs from 'fs'
+
+const METADATA_DIRNAME = '.gaia-metadata'
+
+export class GaiaDiskReader {
+
+  config: Object
+
+  constructor(config: Object) {
+    this.config = config
+  }
+
+  handleGet(topLevelDir: string, filename: string)
+  : Promise<{ exists: boolean, contentType: ?string } > {
+    const storageRoot = this.config.diskSettings.storageRootDirectory
+    if (!storageRoot) {
+      throw new Error('Misconfiguration: no storage root set')
+    }
+
+    const filePath = Path.join(storageRoot, topLevelDir, filename)
+    try {
+      fs.statSync(filePath)
+    } catch (e) {
+      const ret = { exists: false, contentType: undefined }
+      return Promise.resolve().then(() => ret)
+    }
+
+    const metadataPath = Path.join(storageRoot, METADATA_DIRNAME, topLevelDir, filename)
+    try {
+      const metadataJSON = fs.readFileSync(metadataPath).toString()
+      const metadata = JSON.parse(metadataJSON)
+      const ret = { exists: true, contentType: metadata['content-type'] }
+      return Promise.resolve().then(() => ret)
+    } catch (e) {
+      const ret = { exists: true, contentType: 'application/octet-stream' }
+      return Promise.resolve().then(() => ret)
+    }
+  }
+}