Skip to content

Commit

Permalink
Merge branch 'KETTLE-55'
Browse files Browse the repository at this point in the history 
* KETTLE-55:
  KETTLE-55, KETTLE-49, KETTLE-50, KETTLE-61: Harmonised with upcoming fluid.dataSource definitions and committing for 2.0 release
  • Loading branch information
@amb26
amb26 committed Sep 30, 2020
2 parents 8d0cefb + 9da39c8 commit c0633b2
Show file tree
Hide file tree
Showing 20 changed files with 307 additions and 463 deletions.
12 changes: 12 additions & 0 deletions History.md
View file
@@ -1,5 +1,17 @@
# Version History

## 2.0.0 / 2020-09-30

* KETTLE-55: Adopted Infusion's DataSource infrastructure and factored away duplicate code
* KETTLE-50: `kettle.dataSource.file.moduleTerms` has been decoupled from `kettle.dataSource.file` as `kettle.dataSource.moduleTerms`
* KETTLE-49: Eliminated "readOnlyGrade" system in favour of slightly more logical "writableGrade" system and
contextAwareness definition.
* KETTLE-61: Client-side URL DataSource which is now part of the FLUID-6145 branch
* URL DataSource now follows node's modern WhatWG-oriented naming for URL fields
* **BREAKING CHANGES** Kettle after 2.0.0 is only compatible with releases of Infusion from the FLUID-6145 branch which
are dated later than 2020-09-24. This branch will eventually become the released version of Infusion 3.x.
* Other dependency updates

## 1.16.0 / 2020-08-07

* KETTLE-89: Follow HTTP redirects from URL DataSource
Expand Down
9 changes: 9 additions & 0 deletions README.md
View file
Expand Up @@ -18,6 +18,15 @@ In fact, Kettle's dependency on express itself is minimal, since the entirety of
is packaged as a single piece of express-compatible middleware – Kettle could be deployed against any other consumer
of middleware or even a raw node.js HTTP server.

## Notes on Kettle 2.x releases

The 2.x line of Kettle releases are (at the time of writing) still code-compatible with the 1.x releases in terms of
support for user code, but the 2.x Kettle releases break compatibility with older versions of Infusion. Kettle 1.x
has been compatible with mainline releases of Infusion as well as those from in-progress FLUID-6145 and FLUID-6148
branches. Kettle 2.x releases are only compatible with releases of Infusion from the FLUID-6145 branch, newer
than and including 3.0.0-dev.20200930T151056Z.d0b9e348d.FLUID-6145 . The contents of this branch will in time
contribute to the upcoming 3.x releases of Infusion.

## Contents of this repository

### Core Kettle implementation
Expand Down
2 changes: 1 addition & 1 deletion docs/DataSources.md
View file
Expand Up @@ -113,7 +113,7 @@ We document these configuration options in the next section:
<td><code>writeMethod</code></td>
<td><code>String</code> (default: <code>PUT</code>)</td>
<td>The HTTP method to be used when the <code>set</code> method is operated on this writable DataSource
(with <code>writable: true</code>). This defaults to <code>PUT</code> but
(with grade <code>fluid.dataSource.writable</code>). This defaults to <code>PUT</code> but
<code>POST</code> is another option. Note that this option can also be supplied within the
<code>options</code> argument to the <code>set</code> method itself.</td>
</tr>
Expand Down
14 changes: 12 additions & 2 deletions lib/KettleServer.js
View file
Expand Up @@ -55,6 +55,7 @@ fluid.defaults("kettle.server", {
}
},
members: {
listeningPromise: "@expand:fluid.promise()",
expressApp: "@expand:kettle.server.makeExpressApp()",
httpServer: "@expand:kettle.server.httpServer({that}.expressApp)",
dispatcher: "@expand:kettle.server.getDispatcher({that})",
Expand All @@ -70,9 +71,12 @@ fluid.defaults("kettle.server", {
onStopped: null
},
listeners: {
// TODO: Scoping this to a server is not appropriate since in practice Infusion logging is purely stateful
// We need a better, IoC-directed logging framework
"onCreate.setLogging": {
funcName: "fluid.setLogging",
args: "{that}.options.logging"
args: "{that}.options.logging",
priority: "first"
},
"onCreate.contributeMiddleware": {
func: "{that}.events.onContributeMiddleware.fire",
Expand All @@ -94,7 +98,13 @@ fluid.defaults("kettle.server", {
args: "{that}",
priority: "after:registerRouteHandlers"
},
onListen: "{that}.trackConnections",
"onCreate.trackConnections": {
func: "{that}.trackConnections"
},
"onListen.forwardPromise": {
func: "{that}.listeningPromise.resolve",
args: "{that}"
},
onDestroy: "{that}.stop",
beforeStop: "{that}.closeConnections",
onStopped: "kettle.server.shred({that})"
Expand Down
2 changes: 1 addition & 1 deletion lib/KettleUtils.js
View file
Expand Up @@ -202,7 +202,7 @@ fluid.defaults("kettle.dataSource.distributeDevTerms", {
gradeNames: ["fluid.component"],
distributeOptions: {
moduleTerms: {
record: "kettle.dataSource.file.moduleTerms",
record: "kettle.dataSource.moduleTerms",
target: "{that kettle.dataSource.file}.options.gradeNames"
}
}
Expand Down
213 changes: 38 additions & 175 deletions lib/dataSource-core.js
View file
Expand Up @@ -17,6 +17,7 @@ var fluid = fluid || require("infusion"),
kettle = fluid.registerNamespace("kettle"),
JSON5 = JSON5 || require("json5");

fluid.require("querystring", require, "kettle.npm.querystring");

/** Some common content encodings - suitable to appear as the "encoding" subcomponent of a dataSource **/

Expand All @@ -41,21 +42,14 @@ fluid.defaults("kettle.dataSource.encoding.JSON5", {
fluid.defaults("kettle.dataSource.encoding.formenc", {
gradeNames: "fluid.component",
invokers: {
parse: "node.querystring.parse({arguments}.0)",
render: "node.querystring.stringify({arguments}.0)"
parse: "kettle.npm.querystring.parse({arguments}.0)",
render: "kettle.npm.querystring.stringify({arguments}.0)"
},
contentType: "application/x-www-form-urlencoded"
});

fluid.defaults("kettle.dataSource.encoding.none", {
gradeNames: "fluid.component",
invokers: {
parse: "fluid.identity",
render: "fluid.identity"
},
contentType: "text/plain"
});

// Patch the core JSON encoding from Infusion to use our JISON parser in Kettle for better diagnostics
fluid.makeGradeLinkage("kettle.dataSource.encoding.linkage.JSON", ["fluid.dataSource.encoding.JSON"], "kettle.dataSource.encoding.JSON");

/** Definitions for parsing JSON using jsonlint to render errors **/

Expand Down Expand Up @@ -149,166 +143,30 @@ kettle.dataSource.stringifyJSON5 = function (obj) {
return obj === undefined ? "" : JSON5.stringify(obj, null, 4);
};

/**
* The head of the hierarchy of dataSource components. These abstract
* over the process of read and write access to data, following a simple CRUD-type semantic, indexed by
a coordinate model (directModel) and which may be asynchronous.
* Top-level methods are:
* get(directModel[, callback|options] - to get the data from data resource
* set(directModel, model[, callback|options] - to set the data (only if writable option is set to `true`)
*/
fluid.defaults("kettle.dataSource", {
gradeNames: ["fluid.component", "{that}.getWritableGrade"],
mergePolicy: {
setResponseTransforms: "replace"
},
events: {
// events "onRead" and "onWrite" are operated in a custom workflow by fluid.fireTransformEvent to
// process dataSource payloads during the get and set process. Each listener
// receives the data returned by the last.
onRead: null,
onWrite: null,
onError: null
},
components: {
encoding: {
type: "kettle.dataSource.encoding.JSON"
}
},
listeners: {
onRead: {
func: "{encoding}.parse",
namespace: "encoding"
},
onWrite: {
func: "{encoding}.render",
namespace: "encoding"
}
},
invokers: {
get: {
funcName: "kettle.dataSource.get",
args: ["{that}", "{arguments}.0", "{arguments}.1"] // directModel, options/callback
},
// getImpl: must be implemented by a concrete subgrade
getWritableGrade: {
funcName: "kettle.dataSource.getWritableGrade",
args: ["{that}", "{that}.options.writable", "{that}.options.readOnlyGrade"]
}
},
// In the case of parsing a response from a "set" request, only transforms of these namespaces will be applied
setResponseTransforms: ["encoding"],
charEncoding: "utf8", // choose one of node.js character encodings
writable: false
});

// TODO: Move this system over to "linkage" too
/* Use the peculiar `readOnlyGrade` member defined on every concrete DataSource to compute the name of the grade that should be
* used to operate its writable variant if the `writable: true` options is set
*/
kettle.dataSource.getWritableGrade = function (that, writable, readOnlyGrade) {
if (!readOnlyGrade) {
fluid.fail("Cannot evaluate writable grade without readOnlyGrade option");
}
if (writable) {
return fluid.model.composeSegments(readOnlyGrade, "writable");
}
};
/** A mixin grade for a dataSource which automatically expands any %terms corresponding to module names registered in Infusion's module database */

fluid.defaults("kettle.dataSource.writable", {
gradeNames: ["fluid.component"],
invokers: {
set: {
funcName: "kettle.dataSource.set",
args: ["{that}", "{arguments}.0", "{arguments}.1", "{arguments}.2"] // directModel, model, options/callback
}
// setImpl: must be implemented by a concrete subgrade
}
fluid.defaults("kettle.dataSource.moduleTerms", {
termMap: "@expand:fluid.module.terms()"
});

// Registers the default promise handlers for a dataSource operation -
// i) If the user has supplied a function in place of method <code>options</code>, register this function as a success handler
// ii) if the user has supplied an onError handler in method <code>options</code>, this is registered - otherwise
// we register the firer of the dataSource's own onError method.

kettle.dataSource.registerStandardPromiseHandlers = function (that, promise, options) {
promise.then(typeof(options) === "function" ? options : null,
options.onError ? options.onError : that.events.onError.fire);
};

/** Apply default members to the options governing this dataSource request. Called when first receiving either a `get` or `set` request to the
* top-level driver.
* @param {Object} componentOptions - The DataSource's component options
* @param {Object} [options] - [optional] Any additional options for this request. If supplied, this object reference will be written to in order to
* assemble the returned options
* @param {Object} directModel - The direct model supplied to the DataSource API for this request
* @param {Booleanish} [isSet] - [optional] `true` if this is a `set` DataSource request
* @return {Object} The fully merged request options in the same object reference as `options` if it was set
*/
kettle.dataSource.defaultiseOptions = function (componentOptions, options, directModel, isSet) {
options = options || {};
options.directModel = directModel;
options.operation = isSet ? "set" : "get";
options.reverse = isSet ? true : false;
options.writeMethod = options.writeMethod || componentOptions.writeMethod || "PUT"; // TODO: parameterise this, only of interest to HTTP DataSource
options.notFoundIsEmpty = options.notFoundIsEmpty || componentOptions.notFoundIsEmpty;
return options;
};

// TODO: Strategy note on these core engines - we need/plan to remove the asymmetry between the "concrete DataSource" (e.g. file or URL) and elements
// of the transform chain. The so-called getImpl/setImpl should be replaced with sources of "just another" transform element, but in this case
// one which transforms out of or into nothing to acquire the initial/final payload.

/** Operate the core "transforming promise workflow" of a dataSource's `get` method. Gets the "initial payload" from the dataSource's `getImpl` method
* and then pushes it through the transform chain to arrive at the final payload.
* @param {Component} that - The dataSource itself
* @param {Object} directModel - The direct model expressing the "coordinates" of the model to be fetched
* @param {Object} [options] - [optional] A structure of options configuring the action of this get request - many of these will be specific to the particular concrete DataSource
* @return {Promise} A promise for the final resolved payload
*/
kettle.dataSource.get = function (that, directModel, options) {
options = kettle.dataSource.defaultiseOptions(that.options, options, directModel);
var initPayload = that.getImpl(options, directModel);
var promise = fluid.promise.fireTransformEvent(that.events.onRead, initPayload, options);
kettle.dataSource.registerStandardPromiseHandlers(that, promise, options);
return promise;
};

/** Operate the core "transforming promise workflow" of a dataSource's `set` method. Pushes the user's payload backwards through the
* transforming promise chain (in the opposite direction to that applied on `get`, and then applies it to the dataSource's `setImpl` method.
* Any return from this is then pushed forwards through a limited range of the transforms (typically, e.g. just decoding it as JSON)
* on its way back to the user.
* @param {Component} that - The dataSource itself
* @param {Object} directModel - The direct model expressing the "coordinates" of the model to be written
* @param {Object} model - The payload to be written to the dataSource
* @param {Object} [options] - [optional] A structure of options configuring the action of this set request - many of these will be specific to the particular concrete DataSource
* @return {Promise} A promise for the final resolved payload (not all DataSources will provide any for a `set` method - the semantic of this is DataSource specific)
*/
kettle.dataSource.set = function (that, directModel, model, options) {
options = kettle.dataSource.defaultiseOptions(that.options, options, directModel, true); // shared and writeable between all participants
var transformPromise = fluid.promise.fireTransformEvent(that.events.onWrite, model, options);
var togo = fluid.promise();
transformPromise.then(function (transformed) {
var innerPromise = that.setImpl(options, directModel, transformed);
innerPromise.then(function (setResponse) { // Apply limited transforms to a SET response payload
var options2 = kettle.dataSource.defaultiseOptions(that.options, fluid.copy(options), directModel);
options2.filterNamespaces = that.options.setResponseTransforms;
var retransformed = fluid.promise.fireTransformEvent(that.events.onRead, setResponse, options2);
fluid.promise.follow(retransformed, togo);
}, function (error) {
togo.reject(error);
});
});
kettle.dataSource.registerStandardPromiseHandlers(that, togo, options);
return togo;
};


/**
* A mixin grade for a data source suitable for communicating with the /{db}/{docid} URL space of CouchDB for simple CRUD-style reading and writing
*/

fluid.defaults("kettle.dataSource.CouchDB", {
// Link on to the existing writable: true flag maintained by a core fluid.dataSource
contextAwareness: {
writableCouchDB: {
checks: {
writableCouchDB: {
contextValue: "{fluid.dataSource}.options.writable",
// Note that it would be preferable for gradeNames to form a hash as in FLUID-6439 so that we could
// override it selectively rather than to duplicate the entire contextAwareness definition as here
gradeNames: "kettle.dataSource.CouchDB.writable"
}
}
}
},
mergePolicy: {
"rules": "nomerge"
},
Expand All @@ -323,26 +181,31 @@ fluid.defaults("kettle.dataSource.CouchDB", {
listeners: {
onRead: {
funcName: "kettle.dataSource.CouchDB.read",
args: ["{that}", "{arguments}.0"], // resp
args: ["{that}", "{arguments}.0"], // response
namespace: "CouchDB",
priority: "after:encoding"
}
}
});

/**
* A pure mixin grade operating write logic for a CouchDB-backed dataSource. This performs a "read before write" strategy
* in order to minimise the possibility of a write conflict, at the risk of overwriting a previous update. This grade
* is configured automatically when the "writable: true" option is supplied to a "kettle.dataSource.CouchDB" source and
* should not be configured directly by the user.
*/

fluid.defaults("kettle.dataSource.CouchDB.writable", {
listeners: {
onWrite: {
funcName: "kettle.dataSource.CouchDB.write",
args: ["{that}", "{arguments}.0", "{arguments}.1"], // model, options
namespace: "CouchDB",
priority: "after:encoding"
priority: "before:encoding"
}
}
});

fluid.makeGradeLinkage("kettle.dataSource.CouchDB.linkage", ["kettle.dataSource.writable", "kettle.dataSource.CouchDB"], "kettle.dataSource.CouchDB.writable");

/**
* Convert a dataSource payload from CouchDB-encoded form -
*
Expand All @@ -351,25 +214,25 @@ fluid.makeGradeLinkage("kettle.dataSource.CouchDB.linkage", ["kettle.dataSource.
* ii) Transform the output from CouchDB using `that.options.rules.readPayload`. The default rules reverse the default
* "value" encoding used by `kettle.dataSource.CouchDB.write` (see below).
* @param {Component} that - The dataSource component, used to read the payload read transform option
* @param {Object} resp - JSON-parsed response as received from CouchDB
* @param {Object} response - JSON-parsed response as received from CouchDB
* @return {Object} The transformed return payload
*/
kettle.dataSource.CouchDB.read = function (that, resp) {
kettle.dataSource.CouchDB.read = function (that, response) {
// if undefined, pass that through as per dataSource (just for consistency in FS-backed tests)
var togo;
if (resp === undefined) {
if (response === undefined) {
togo = undefined;
} else {
if (resp.error) {
if (response.error) {
var error = {
isError: true,
statusCode: resp.statusCode,
message: resp.error + ": " + resp.reason
statusCode: response.statusCode,
message: response.error + ": " + response.reason
};
togo = fluid.promise();
togo.reject(error);
} else {
togo = fluid.model.transformWithRules(resp, that.options.rules.readPayload);
togo = fluid.model.transformWithRules(response, that.options.rules.readPayload);
}
}
return togo;
Expand All @@ -388,7 +251,7 @@ kettle.dataSource.CouchDB.read = function (that, resp) {
kettle.dataSource.CouchDB.write = function (that, model, options) {
var directModel = options.directModel;
var doc = fluid.model.transformWithRules(model, that.options.rules.writePayload);
var original = that.get(directModel, {filterNamespaces: ["encoding"], notFoundIsEmpty: true});
var original = that.get(directModel, {filterNamespaces: ["impl", "encoding"], notFoundIsEmpty: true});
var togo = fluid.promise();
original.then(function (originalDoc) {
if (originalDoc) {
Expand Down

0 comments on commit c0633b2

Please sign in to comment.