Merge 39d1c5a into 1784fd1

.eslintrc.yml

@@ -1 +1,30 @@
-extends: 'eslint-config-node-services'
+extends: 'eslint-config-wikimedia'
+
+parserOptions:
+  ecmaVersion: 10
+
+env:
+  node: true
+
+rules:
+  indent:
+    - error
+    - 4
+    - SwitchCase: 1
+      MemberExpression: off
+  space-in-parens: [error, never]
+  no-multi-spaces: off
+  no-multiple-empty-lines: off
+  one-var: off
+  array-bracket-spacing: off
+  computed-property-spacing: off
+  no-unused-vars: [error, {args: none}]
+  no-underscore-dangle: off
+  comma-dangle: off
+  camelcase: off
+  key-spacing: off
+  space-before-function-paren: off
+  quote-props: off
+  operator-linebreak: off
+  max-statements-per-line: off
+  brace-style: off

.gitignore

@@ -2,5 +2,11 @@ Dockerfile
 .idea/
 coverage
 config.yaml
-node_modules
+node_modules**
 npm-debug.log
+*.vscode
+*.code-workspace
+package-lock.json
+.eslintcache
+.nyc_output
+

.nycrc

@@ -0,0 +1,13 @@
+{
+    "exclude": [
+        "test/**",
+        "lib/util.js",
+        "lib/kafka.js",
+        "lib/swagger-ui.js",
+        "app.js"
+    ],
+    "reporter": [
+        "text",
+        "html"
+    ]
+}

.travis.yml

@@ -1,8 +1,7 @@
-language: node_js
-
-sudo: false
+dist: xenial
 
+language: node_js
 node_js:
-  - "6"
-  - "8"
-  - "node"
+  - "10"
+
+after_success: npm run coveralls

app.js

@@ -8,7 +8,6 @@ const compression = require('compression');
 const bodyParser = require('body-parser');
 const fs = BBPromise.promisifyAll(require('fs'));
 const sUtil = require('./lib/util');
-const apiUtil = require('./lib/api-util');
 const packageInfo = require('./package.json');
 const yaml = require('js-yaml');
 const addShutdown = require('http-shutdown');
@@ -31,6 +30,13 @@ function initApp(options) {
     app.conf = options.config;      // this app's config options
     app.info = packageInfo;         // this app's package info
 
+    // --- BEGIN EventGate modification ---
+    // Pass service-runners determined appBasePath into
+    // the application config so that it can be used to
+    // more intelligently load eventgate_factory_modules.
+    app.conf.app_base_path = options.appBasePath;
+    // --- END EventGate modification ---
+
     // ensure some sane defaults
     if (!app.conf.port) { app.conf.port = 8888; }
     if (!app.conf.interface) { app.conf.interface = '0.0.0.0'; }
@@ -66,9 +72,6 @@ function initApp(options) {
         return item.trim();
     }).join('|')})$`, 'i');
 
-    // set up the request templates for the APIs
-    apiUtil.setupApiTemplates(app);
-
     // set up the spec
     if (!app.conf.spec) {
         app.conf.spec = `${__dirname}/spec.yaml`;
@@ -124,8 +127,15 @@ function initApp(options) {
     app.set('etag', false);
     // enable compression
     app.use(compression({ level: app.conf.compression_level }));
-    // use the JSON body parser
-    app.use(bodyParser.json());
+    // --- BEGIN EventGate modification ---
+    // use the JSON body parser.
+    // We specify both the correct type 'application/json', as well as
+    // 'text/plain', so that browser navigator.sendBeacon calls will work.
+    // sendBeacon CORS restrictions don't allow sendBeacon to POST with an application/json
+    // content type, so we allow its default text/plain to be parsed as JSON.
+    // See: https://stackoverflow.com/a/44142982
+    app.use(bodyParser.json({ limit: app.conf.max_body_size || '100kb', type: ['application/json', 'text/plain']  }));
+    // --- END EventGate modification ---
     // use the application/x-www-form-urlencoded parser
     app.use(bodyParser.urlencoded({ extended: true }));
 
@@ -137,6 +147,7 @@ function initApp(options) {
 /**
  * Loads all routes declared in routes/ into the app
  * @param {Application} app the application object to load routes into
+ * @param {string} dir
  * @return {bluebird} a promise resolving to the app object
  */
 function loadRoutes(app, dir) {
@@ -212,7 +223,7 @@ function createServer(app) {
         // Don't delay incomplete packets for 40ms (Linux default) on
         // pipelined HTTP sockets. We write in large chunks or buffers, so
         // lack of coalescing should not be an issue here.
-        server.on("connection", (socket) => {
+        server.on('connection', (socket) => {
             socket.setNoDelay(true);
         });
 
@@ -227,6 +238,8 @@ function createServer(app) {
  * options and the logger- and metrics-reporting objects from
  * service-runner and starts an HTTP server, attaching the application
  * object to it.
+ * @param {Object} options
+ * @return {Function}
  */
 module.exports = function(options) {
 

config.dev.yaml

@@ -1,77 +1,51 @@
+# EventGate example config file.
+#
+# Configures the service-runner as well as the EventGate application.
+# See: https://github.com/wikimedia/service-runner#config-loading
+
 # Number of worker processes to spawn.
 # Set to 0 to run everything in a single process without clustering.
 # Use 'ncpu' to run as many workers as there are CPU units
 num_workers: 0
 
 # Log error messages and gracefully restart a worker if v8 reports that it
 # uses more heap (note: not RSS) than this many mb.
-worker_heap_limit_mb: 250
+worker_heap_limit_mb: 200
 
 # Logger info
 logging:
-  level: trace
-#  streams:
-#  # Use gelf-stream -> logstash
-#  - type: gelf
-#    host: logstash1003.eqiad.wmnet
-#    port: 12201
-
-# Statsd metrics reporter
-metrics:
-  #type: log
-  #host: localhost
-  #port: 8125
+  level: info
 
 services:
-  - name: service-template-node
+  - name: eventgate-dev
     # a relative path or the name of an npm package, if different from name
     module: ./app.js
     # optionally, a version constraint of the npm package
     # version: ^0.4.0
     # per-service config
     conf:
-      port: 6927
-      # interface: localhost # uncomment to only listen on localhost
+      port: 8192
+      # Events can be large; increase max body size
+      max_body_size: 4mb
+
       # more per-service config settings
-      # the location of the spec, defaults to spec.yaml if not specified
-      spec: ./spec.template.yaml
-      # allow cross-domain requests to the API (default '*')
-      cors: '*'
-      # to disable use:
-      # cors: false
-      # to restrict to a particular domain, use:
-      # cors: restricted.domain.org
-      # content for the CSP headers
-      # csp: false  # uncomment this line to disable sending them
-      # URL of the outbound proxy to use (complete with protocol)
-      # proxy: http://my.proxy.org:8080
-      # the list of domains for which not to use the proxy defined above
-      # no_proxy_list:
-      #   - domain1.com
-      #   - domain2.org
-      # the list of incoming request headers that can be logged; if left empty,
-      # the following headers are allowed: cache-control, content-length,
-      # content-type, if-match, user-agent, x-request-id
-      # log_header_whitelist:
-      #   - cache-control
-      #   - content-length
-      #   - content-type
-      #   - if-match
-      #   - user-agent
-      #   - x-request-id
-      # the user agent to use when issuing requests
-      # user_agent: service-template-node
-      # the template used for contacting the MW API
-      mwapi_req:
-        method: post
-        uri: https://{{domain}}/w/api.php
-        headers:
-          user-agent: '{{user-agent}}'
-        body: '{{ default(request.query, {}) }}'
-      # the template used for contacting RESTBase
-      restbase_req:
-        method: '{{request.method}}'
-        uri: https://{{domain}}/api/rest_v1/{+path}
-        query: '{{ default(request.query, {}) }}'
-        headers: '{{request.headers}}'
-        body: '{{request.body}}'
+      user_agent: eventgate-dev
+
+      eventgate_factory_module: '../lib/factories/dev-eventgate'
+
+      # This field in each event will be used to extract a
+      # (possibly relative) schema uri.  The default is $schema.
+      # An array of field names will cause EventGate to search for
+      # fields by these names in each event, using the first match.
+      schema_uri_field: $schema
+
+      # If set, these URIs will be prepended to any relative schema URI
+      # extracted from each event's schema_field.  The resulting URLs will
+      # be searched until a schema is found.  Change this
+      # to match paths to your local schema repositories.
+      schema_base_uris: [
+          https://schema.wikimedia.org/repositories/primary/jsonschema,
+          https://schema.wikimedia.org/repositories/secondary/jsonschema
+      ]
+
+    #   output_path: ./output.json