Merge pull request #822 from null-a/file-param-store

Add file-based param store backend

docs/optimization/filestore.rst

@@ -0,0 +1,31 @@
+.. highlight:: none
+
+Persistence
+===========
+
+The file store provides a simple way to persist :ref:`parameters
+<parameters>` across executions. Parameters are read from a file
+before the program is executed, and written back to the file once the
+program finishes. Enable it like so::
+
+  webppl model.wppl --param-store file --param-id my-parameters
+
+The file used takes its name from the ``param-id`` command line
+argument (appended with ``.json``) and is expected to be located in
+the current directory. A new file will be created if this file does
+not already exist.
+
+An alternative directory can be specified using the
+``WEBPPL_PARAM_PATH`` environment variable.
+
+A random file name is generated when the ``param-id`` argument is
+omitted.
+
+Parameters are also periodically written to the file during
+:ref:`optimization <optimization>`. The frequency of writes can be
+controlled using the ``WEBPPL_PARAM_INTERVAL`` environment variable.
+This specifies the minimum amount of time (in milliseconds) that
+should elapse between writes. The default is 10 seconds.
+
+Note that this is not intended for parallel use. The :ref:`mongo store
+<async>` should be used for this instead.

docs/optimization/index.rst

@@ -43,4 +43,5 @@ optimization <optimize>`, primitives for specifying :ref:`parameters
 
    optimize
    parameters
+   filestore
    async

docs/optimization/parameters.rst

@@ -1,8 +1,8 @@
+.. _parameters:
+
 Parameters
 ==========
 
-.. _parameters:
-
 .. js:function:: param([options])
 
    Retrieves the value of a parameter by name. The parameter is

src/params/config.js

@@ -3,6 +3,7 @@
 var _ = require('lodash');
 var memoryStore = require('./store/memory');
 var mongoStore = require('./store/mongo');
+var fileStore = require('./store/file');
 
 
 // Id of active parameter table
@@ -38,7 +39,8 @@ function isManualId() {
 
 var stores = {
   memory: memoryStore,
-  mongo: mongoStore
+  mongo: mongoStore,
+  file: fileStore
 };
 
 var _store = memoryStore;

src/params/store/file.js

@@ -0,0 +1,92 @@
+'use strict';
+
+var assert = require('assert');
+var fs = require('fs');
+var join = require('path').join;
+var _ = require('lodash');
+var serialize = require('../serialize');
+var util = require('../../util');
+
+var path = _.get(process.env, 'WEBPPL_PARAM_PATH', '.');
+var interval = parseInt(_.get(process.env, 'WEBPPL_PARAM_INTERVAL', 1e4));
+var verbose = parseInt(_.get(process.env, 'WEBPPL_PARAM_VERBOSE', 0));
+
+// Holds a {params, timestamp} object per parameter set id.
+var store = {};
+
+function filename(id) {
+  assert.ok(_.isString(path), 'Expected path to be defined.');
+  return join(path, id + '.json');
+}
+
+function read(id) {
+  try {
+    var params = serialize.deserializeParams(fs.readFileSync(filename(id)));
+    if (verbose) {
+      console.log('Read parameter set ' + id + '.');
+    }
+    return params;
+  } catch (e) {
+    if (verbose) {
+      console.log('No file found for parameter set ' + id +
+                  '. Using empty parameter set.');
+    }
+    return {};
+  }
+}
+
+function write(params, id) {
+  try {
+    fs.writeFileSync(filename(id), serialize.serializeParams(params));
+    if (verbose) {
+      console.log('Wrote parameter set ' + id + '.');
+    }
+  } catch (e) {
+    util.warn('Error writing parameter set ' + id + '.');
+  }
+}
+
+// External interface:
+
+function start(k) {
+  // Ensure the parameter directory (currently) exists, error
+  // otherwise.
+  fs.accessSync(path);
+  return k();
+}
+
+// Write all parameters to disk after the program completes.
+function stop(k) {
+  _.forEach(store, function(obj, id) {
+    write(obj.params, id);
+  });
+  return k();
+}
+
+function getParams(id, k) {
+  if (!_.has(store, id)) {
+    store[id] = {params: read(id), timestamp: Date.now()};
+  }
+  return k(store[id].params);
+}
+
+// Perform throttled writes to disk when Optimize updates parameters.
+// This makes to possible to recover progress should the program crash
+// before the final write happens. The frequency of writes can be
+// controlled using an environment variable.
+function setParams(id, params, k) {
+  store[id].params = params;
+  var now = Date.now();
+  if (now - store[id].timestamp > interval) {
+    write(params, id);
+    store[id].timestamp = now;
+  }
+  return k();
+}
+
+module.exports = {
+  start: start,
+  stop: stop,
+  getParams: getParams,
+  setParams: setParams
+};