Clean master commit

.gitignore

@@ -0,0 +1,10 @@
+/dist
+/logs
+/npm-debug.log
+/node_modules
+/temp
+.DS_Store
+*.swp
+
+*service-account-key\.json
+src/config.js

Dockerfile

@@ -0,0 +1,20 @@
+FROM mhart/alpine-node:10.11
+
+LABEL authors="Lachlan Kermode <lk@forensic-architecture.org>"
+
+# Install app dependencies
+COPY package.json /www/package.json
+RUN cd /www; yarn
+
+# Copy app source
+COPY . /www
+WORKDIR /www
+RUN yarn build
+RUN mkdir -p temp
+
+# set your port
+ENV PORT 8080
+EXPOSE  8080
+
+# start command as per package.json
+CMD ["yarn", "start"]

LICENSE

@@ -0,0 +1,20 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 Jason Miller
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

README.md

@@ -0,0 +1,148 @@
+<h1 align="center">
+  Datasheet Server
+</h1>
+
+<p align="center">
+  <strong>Turn spreadsheet data into a structured, dynamic API. </strong><br>
+</p>
+<!-- <p align="center">
+  <a href="https://github.com/gatsbyjs/gatsby/blob/master/LICENSE">
+    <img src="https://img.shields.io/badge/license-MIT-blue.svg" alt="Gatsby is released under the MIT license." />
+  </a>
+  <a href="#configru">
+    <img src="https://circleci.com/gh/gatsbyjs/gatsby.svg?style=shield" alt="Current CircleCI build status." />
+  </a>
+  <a href="https://www.npmjs.org/package/gatsby">
+    <img src="https://img.shields.io/npm/v/gatsby.svg" alt="Current npm package version." />
+  </a>
+  <a href="https://npmcharts.com/compare/gatsby?minimal=true">
+    <img src="https://img.shields.io/npm/dm/gatsby.svg" alt="Downloads per month on npm." />
+  </a>
+  <a href="https://gatsbyjs.org/docs/how-to-submit-a-pr/">
+    <img src="https://img.shields.io/badge/PRs-welcome-brightgreen.svg" alt="PRs welcome!" />
+  </a>
+</p> -->
+
+<h3 align="center">
+  <a href="#overview">Overview</a>
+  <span> · </span>
+  <a href="#configuration">Configuration</a>
+  <span> · </span>
+  <a href="#quickstart">Quickstart</a>
+</h3>
+
+Datasheet server makes resources from a spreadsheet available as a structured API.
+
+- **Manage structured data without developers**. Allows anyone to dynamically manage data, while simultaneously making this data available in a reliably structured format for frontend interfaces and other programmatic applications.
+- **Designed for a dynamic workflow**. References data in a spreadsheet source as a ground truth, but adds a layer of indirection that keep API routes from breaking when changes are made.
+- **Customisable data transformation**. Easily create new blueprints to specify the API structure that should be presented from source data.
+- **Extensible architecture**. Currently supports Google Sheet as a source and a REST-like query language, but structured modularly with an intention to support other sources and query languages.
+
+## [Overview](#overview)
+Datasheet server is a Node server developed at [Forensic Architecture](https://forensic-architecture.org) to make data that is being dynamically modified by researchers concurrently consumable for programmatic applications as an API. We use spreadsheets extensively to keep track of information during [our investigations](http://forensic-architecture.org/cases), and we often want to make this data available via structured queries to make it available in a frontend web application, a game engine, or another use case.
+
+Querying data directly from spreadsheets is brittle, as it relies on the maintenance of a rigid structure in the sheets at all times. By putting Datasheet Server as a proxy that sits in between source sheets and their consumers, it is possible to dynamically modify sheets without breaking applications. A data admin can then use Datasheet Server to ensure that applications always receive eligible data, without foregoing the spreadsheets as sources of truth.  
+
+### Design Concepts
+The codebase currently only supports Google Sheets as a source, and a REST-like format as a query language. It is designed, however, with extensibility in mind.
+
+**Sources**     
+A source represents a sheet-like collection of data, such as a Google Sheet. A source has one or more **tabs**, each of which contains a 2-dimensional grids of cells. Each cell contains a body of text (a string).
+
+**Resources**     
+The data from sources are made available as resources, which are structured blocks of data that are granularly accessible through a query language. Resources are the outfacing aspect of Datasheet Server, and represent the only kind of data that can be queried by applications. Each resource is configured with one or more **query languages**. (Currently only a REST-like query language supported.)
+
+**Blueprints**       
+Blueprints are a data structure that represent the way that infromation from **sources** are to be turned into **resources**. For each tab in a source, there is a corresponding Blueprint. Blueprints are created through a [blueprinter function](/src/blueprinters) invoked on the raw data from a source tab.
+
+Blueprints are JSON objects. There have two forms:
+
+1. _desaturated_ -- describes the resources and query languages available on data from a source tab.
+2. _saturated_ --  both describes resources available on data from a source etab, and contains that data.
+
+A desaturated Blueprint can be saturated by retrieving its data from the server's **model layer**, which stores tab data from sources.
+
+A JSON catalogue of the available blueprints (desaturated) in a server is available at `/api/blueprints`.
+
+## [Configuration](#configuration)
+Copy the [example.config.js](/src/example.config.js) in the [src](/src) directory into a file named 'config.js'. Modify the options in this file accordingly:
+
+| Option  | Description | Type |
+| ------- | ----------- | ---- |
+| port | The port at which the server will make data available.  | integer |
+| googleSheets  | The configuration object for [Google Sheet](https://www.google.co.uk/sheets/about/) data sources. See the [Sources](#source-google-sheets) section below. | object |
+
+#### [Sources](#sources)
+###### [Google Sheets](#source-google-sheets)
+In order to make the data from a Sheet accessible to the server, you need to [create a service account](https://cloud.google.com/iam/docs/creating-managing-service-accounts). Once created, give the service account email access to each Sheet from which you want to serve data. ('View Only' access is sufficient, as the server never modifies data.)
+
+| Option | Description | Type |
+| ------ | ----------- | ---- |
+| email | The email address of the service account. This is available in the downloadable service account JSON in the `client_email` field. | string |
+| privateKey | The private key associated with the service account. This is available in the downloadable service account JSON in the `private_key` field. | string |
+| sheets | A list of objects, one for each sheet that is being used as a source. Each sheet object has a `name` (String), an `id` (String), and a `tabs` (object) field, which are explained below. | object |
+
+Each Google Sheet being used as a as source requires a corresponding object in `sheets`. The object should be structured as follows:
+
+| Option | Description | Type |
+| ------ | ----------- | ---- |
+| name | Used to refer to data served from this source | string |
+| id | The ID of the sheet in Google. (You can find it in the address bar when the Sheet is open in a browser. It is the string that follows 'spreadsheets/d/'). | string |
+| tabs | An object that maps each tab in the source to a Blueprinter. All of the Blueprinters in the [blueprinters folder](/lib/blueprinters) are available through a single import as at the top of [example.config.js](/src/example.config.js). <br> To correctly associate a Blueprinter, the object key needs to be _the tab name with all lowercase letters, and spaces replaced by a '-'_. For example, if the tab name in Google Sheets is 'Info About SHEEP', the object key should be 'info-about-sheep'. <br> The value should be the Blueprinter function that you want to use for the data in that tab. See the example of a configuration object below. <br>TODO: no Blueprinter is used by default. | object |
+
+###### Example Configuration Object
+```js
+import BP from './lib/blueprinters'
+
+export default {
+  port: 4040,
+  googleSheets: {
+    email: 'project-name@reliable-baptist-23338.iam.gserviceaccount.com',
+    privateKey: 'SOME_PRIVATE_KEY',
+    sheets: [
+      {
+        name: 'example',
+        id: '1s-vfBR8Uy-B-TLO_C5Ozw4z-L0E3hdP8ohMV761ouRI',
+        tabs: {
+          'objects': BP.byRow,
+        }
+      },
+    ]
+  }
+}
+
+```
+
+
+## [Quickstart](#quickstart)
+Clone the repository to your local:
+```
+git clone https://www.github.com/forensic-architecture/datasheet-server
+```
+### Run with Docker
+To create a new instance of the server with [Docker](https://www.docker.com/) installed, clone the repository, create a `config.js`, and build the image:
+```sh
+docker build -t datasheet-server .
+```
+You can then run the container and make available the relevant port (`4040` by default):
+```sh
+docker run -d -p 4040:4040 datasheet-server
+```
+If running on a cloud server, you'll probably need to zero out the host IP:
+```sh
+docker run -d -p 0.0.0.0:4040:4040 datasheet-server
+```
+
+### Run locally
+Install dependencies:
+```sh
+yarn # npm install
+```
+And run development server:
+```sh
+yarn dev # npm run dev
+```
+License
+-------
+
+MIT

package.json

@@ -0,0 +1,75 @@
+{
+  "name": "grenfell-server",
+  "version": "0.3.0",
+  "description": "Starter project for an ES6 RESTful Express API",
+  "main": "dist",
+  "scripts": {
+    "dev": "nodemon -w src --exec \"babel-node src\"",
+    "build": "npx babel src -d dist",
+    "start": "node dist",
+    "lint": "eslint src",
+    "test": "ava --watch"
+  },
+  "eslintConfig": {
+    "extends": "eslint:recommended",
+    "parserOptions": {
+      "ecmaVersion": 7,
+      "sourceType": "module"
+    },
+    "env": {
+      "node": true
+    },
+    "rules": {
+      "no-console": 0,
+      "no-unused-vars": 1
+    }
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/developit/express-es6-rest-api.git"
+  },
+  "author": "Jason Miller <jason@developit.ca>",
+  "license": "MIT",
+  "dependencies": {
+    "body-parser": "^1.13.3",
+    "compression": "^1.5.2",
+    "cors": "^2.7.1",
+    "express": "^4.13.3",
+    "express-graphql": "^0.6.12",
+    "googleapis": "^32.0.0",
+    "graphql": "^0.13.2",
+    "morgan": "^1.8.0",
+    "mz": "^2.7.0",
+    "node-fetch": "^2.2.0",
+    "object-hash": "^1.3.0",
+    "ramda": "^0.25.0",
+    "resource-router-middleware": "^0.6.0"
+  },
+  "devDependencies": {
+    "@babel/cli": "^7.1.2",
+    "@babel/core": "^7.1.2",
+    "@babel/node": "^7.0.0",
+    "@babel/preset-env": "^7.1.0",
+    "@babel/register": "^7.0.0",
+    "ava": "1.0.0-beta.8",
+    "eslint": "^3.1.1",
+    "nodemon": "^1.9.2"
+  },
+  "babel": {
+    "presets": [
+      "@babel/preset-env"
+    ]
+  },
+  "ava": {
+    "files": [
+      "test/**/*.js"
+    ],
+    "require": [
+      "@babel/register"
+    ]
+  },
+  "bugs": {
+    "url": "https://github.com/developit/express-es6-rest-api/issues"
+  },
+  "homepage": "https://github.com/developit/express-es6-rest-api#readme"
+}

src/api/index.js

@@ -0,0 +1,57 @@
+import {version} from "../../package.json";
+import {Router} from "express";
+import {idxSearcher} from "../lib/util";
+
+export default ({config, controller}) => {
+  let api = Router();
+
+  api.get("/", (req, res) => {
+    res.json({
+      version
+    });
+  });
+
+  api.get("/blueprints", (req, res) => {
+    res.json(controller.blueprints());
+  });
+
+  api.get("/:source/:tab/:resource/:frag", (req, res) => {
+    const {source, tab, resource, frag} = req.params;
+    controller
+      .retrieveFrag(source, tab, resource, frag)
+      .then(data => res.json(data))
+      .catch(err =>
+        res.json({
+          error: err.message
+        })
+      );
+  });
+
+  api.get("/:source/:tab/:resource", (req, res) => {
+    controller
+      .retrieve(req.params.source, req.params.tab, req.params.resource)
+      .then(data => res.json(data))
+      .catch(err =>
+        res.json({
+          error: err.message
+        })
+      );
+  });
+
+  api.get("/update", (req, res) => {
+    controller
+      .update()
+      .then(msg =>
+        res.json({
+          success: msg
+        })
+      )
+      .catch(err =>
+        res.json({
+          error: err.message
+        })
+      );
+  });
+
+  return api;
+};

src/blueprinters/byColumn.js

@@ -0,0 +1,34 @@
+import {defaultBlueprint, defaultRoute} from "../lib/blueprinters";
+
+/**
+ * byColumn - generate a Blueprint from a data source by column. Each column
+ * name is a resource, and all values in that column are the resource items.
+ *
+ * @param  {type} data - list of lists representing sheet data.
+ * @return {type} Blueprint
+ * generated.
+ */
+export default function byColumn(tabName, sourceName, sourceId, data) {
+  // Define Blueprint props
+  const bp = R.clone(defaultBlueprint);
+  bp.source = {
+    name: sourceName,
+    id: sourceId
+  };
+  bp.name = tabName;
+
+  // column names define routes
+  const labels = data[0];
+  labels.forEach(label => {
+    bp.routes[label] = R.clone(defaultRoute);
+  });
+
+  // remaining rows as data
+  data.forEach((row, idx) => {
+    if (idx == 0) return;
+    labels.forEach((label, idx) => {
+      bp.routes[label].data.push(row[idx]);
+    });
+  });
+  return bp;
+}