create projections and populations to generate ingest data for elasti…

…csearch

src/buildBackend.js

@@ -13,8 +13,14 @@ module.exports = schemas => {
   // generate elasticsearch mappings
   const elastic_mappings = elasticsearch.generateElasticMappings(schemas)
 
+  // generate projections and populations necessary for search indexing
+  const elastic_projections = elasticsearch.generateElasticProjections(schemas)
+  const elastic_populations = elasticsearch.generateElasticPopulations(schemas)
+
   return {
     mongoose_models,
-    elastic_mappings
+    elastic_mappings,
+    elastic_projections,
+    elastic_populations
   }
 }

src/elasticsearch/generatePopulations.js → src/elasticsearch/generateElasticPopulations.js

@@ -1,16 +1,20 @@
 const processReference = (ref, path, schemas, generatedPopulations, pendingPopulations) => {
   if (schemas[ref].class === 'embedded') {
     Object.keys(schemas[ref].fields).forEach(fieldKey => {
-      if (schemas[ref].fields[fieldKey].type === 'reference' && ref !== schemas[ref].fields[fieldKey].ref) {
+      if (schemas[ref].fields[fieldKey].type === 'reference') {
         processReference(schemas[ref].fields[fieldKey].ref, `${path}.${fieldKey}`, schemas, generatedPopulations, pendingPopulations)
+      } else if (schemas[ref].fields[fieldKey].type === 'array' && schemas[ref].fields[fieldKey].item.type === 'reference') {
+        processReference(schemas[ref].fields[fieldKey].item.ref, `${path}.${fieldKey}`, schemas, generatedPopulations, pendingPopulations)
       }
     })
   } else {
     let population = {
       path,
       model: ref,
       select: Object.keys(schemas[ref].fields).reduce((accumulator, fKey) => {
-        if (!(schemas[ref].fields[fKey].type === 'reference' && schemas[ref].fields[fKey].ref === ref)) {
+        if (!(schemas[ref].fields[fKey].type === 'reference' && schemas[ref].fields[fKey].ref === ref)
+          && !(schemas[ref].fields[fKey].type === 'array' && schemas[ref].fields[fKey].item.type === 'reference' && schemas[ref].fields[fKey].item.ref === ref)
+        ) {
           accumulator[fKey] = 1
         }
 
@@ -30,6 +34,8 @@ const generatePopulations = (schemaKey, schema, schemas, generatedPopulations) =
   return Object.keys(schema.fields).reduce((pendingPopulations, fieldKey) => {
     if (schema.fields[fieldKey].type === 'reference' && schemaKey !== schema.fields[fieldKey].ref) {
       processReference(schema.fields[fieldKey].ref, fieldKey, schemas, generatedPopulations, pendingPopulations)
+    } else if (schema.fields[fieldKey].type === 'array' && schema.fields[fieldKey].item.type === 'reference' && schemaKey !== schema.fields[fieldKey].item.ref) {
+      processReference(schema.fields[fieldKey].item.ref, fieldKey, schemas, generatedPopulations, pendingPopulations)
     }
 
     return pendingPopulations

src/elasticsearch/generateElasticProjections.js

@@ -0,0 +1,21 @@
+module.exports = schemas => {
+  return Object.keys(schemas).reduce((accumulator, schemaKey) => {
+    if (schemas[schemaKey].class === 'collection') {
+      accumulator[schemaKey] = generateProjections(schemaKey, schemas)
+    }
+
+    return accumulator
+  }, {})
+}
+
+const generateProjections = (schemaKey, schemas) => {
+  return Object.keys(schemas[schemaKey].fields).reduce((accumulator, fKey) => {
+    if (!(schemas[schemaKey].fields[fKey].type === 'reference' && schemas[schemaKey].fields[fKey].ref === schemaKey)
+      && !(schemas[schemaKey].fields[fKey].type === 'array' && schemas[schemaKey].fields[fKey].item.type === 'reference' && schemas[schemaKey].fields[fKey].item.ref === schemaKey)
+    ) {
+      accumulator[fKey] = 1
+    }
+
+    return accumulator
+  }, {})
+}

src/elasticsearch/index.js

@@ -1 +1,3 @@
-exports.generateElasticMappings = require('./generateElasticMappings')
+exports.generateElasticMappings = require('./generateElasticMappings')
+exports.generateElasticProjections = require('./generateElasticProjections')
+exports.generateElasticPopulations = require('./generateElasticPopulations')

test/buildBackend.spec.js

@@ -19,6 +19,8 @@ describe('buildBackend()', () => {
 
     expect(backend).to.have.own.property('mongoose_models')
     expect(backend).to.have.own.property('elastic_mappings')
+    expect(backend).to.have.own.property('elastic_projections')
+    expect(backend).to.have.own.property('elastic_populations')
 
   })
 

test/generateElasticPopulations.js

@@ -0,0 +1,201 @@
+const expect = require('chai').expect
+const generateElasticPopulations = require('../src/elasticsearch').generateElasticPopulations
+
+const schemas = {
+  person: {
+    class: 'collection',
+    fields: {
+      name: {
+        type: 'string',
+        required: true,
+        es_indexed: true,
+        es_keyword: true
+      },
+      pokedex: {
+        type: 'reference',
+        required: true,
+        ref: 'pokedex',
+        es_indexed: true
+      },
+      region: {
+        type: 'reference',
+        required: true,
+        ref: 'region',
+        es_indexed: true
+      },
+      visited_regions: {
+        type: 'array',
+        required: true,
+        item: {
+          type: 'reference',
+          required: true,
+          ref: 'region',
+          es_indexed: true
+        }
+      },
+      aliases: {
+        type: 'array',
+        required: false,
+        item: {
+          type: 'reference',
+          required: false,
+          ref: 'person',
+          es_indexed: true
+        }
+      }
+    }
+  },
+
+  pokedex: {
+    class: 'embedded',
+    fields: {
+      pokemon: {
+        type: 'array',
+        required: true,
+        item: {
+          type: 'reference',
+          required: true,
+          ref: 'pokemon',
+          es_indexed: true
+        }
+      },
+      model: {
+        type: 'reference',
+        required: true,
+        ref: 'model',
+        es_indexed: true
+      }
+    }
+  },
+
+  pokemon: {
+    class: 'collection',
+    fields: {
+      name: {
+        type: 'string',
+        required: true,
+        es_indexed: true,
+        es_keyword: true
+      },
+      region: {
+        type: 'reference',
+        required: true,
+        ref: 'region',
+        es_indexed: true
+      },
+      evolves_to: {
+        type: 'reference',
+        required: false,
+        ref: 'pokemon',
+        es_indexed: true
+      }
+    }
+  },
+
+  region: {
+    class: 'collection',
+    fields: {
+      name: {
+        type: 'string',
+        required: true,
+        es_indexed: true,
+        es_keyword: true
+      }
+    }
+  },
+
+  model: {
+    class: 'collection',
+    fields: {
+      name: {
+        type: 'string',
+        required: true,
+        es_indexed: true,
+        es_keyword: true
+      }
+    }
+  }
+}
+
+describe('generateElasticPopulations', () => {
+  it('should list all collection level documents', () => {
+    const populations = generateElasticPopulations(schemas)
+
+    expect(populations).to.have.all.keys('person', 'pokemon', 'region', 'model')
+  })
+
+  it('should generate populations for all collection level fields', () => {
+    const populations = generateElasticPopulations(schemas)
+
+    // person
+    expect(populations.person()).to.be.an('array').that.deep.includes({
+      path: 'region',
+      model: 'region',
+      select: {
+        name: 1
+      }
+    })
+
+    // pokemon
+    expect(populations.pokemon()).to.be.an('array').that.deep.includes({
+      path: 'region',
+      model: 'region',
+      select: {
+        name: 1
+      }
+    })
+
+    // region
+    expect(populations.region()).to.be.empty
+
+    // model
+    expect(populations.model()).to.be.empty
+  })
+
+  it('should generate populations for all array fields whose items are collection level fields', () => {
+    const populations = generateElasticPopulations(schemas)
+
+    // person
+    expect(populations.person()).to.be.an('array').that.deep.includes({
+      path: 'visited_regions',
+      model: 'region',
+      select: {
+        name: 1
+      }
+    })
+  })
+
+  it('should generate nested population paths for fields with collection level references in embedded documents', () => {
+    const populations = generateElasticPopulations(schemas)
+
+    // person
+    expect(populations.person()).to.be.an('array').that.deep.includes({
+      path: 'pokedex.model',
+      model: 'model',
+      select: {
+        name: 1
+      }
+    })
+  })
+
+  it('should generate nested population paths for fields with collection level references inside arrays in embedded documents', () => {
+    const populations = generateElasticPopulations(schemas)
+
+    // person
+    expect(populations.person()).to.be.an('array').that.deep.includes({
+      path: 'pokedex.pokemon',
+      model: 'pokemon',
+      select: {
+        name: 1,
+        region: 1
+      },
+      populate: [{
+        path: 'region',
+        model: 'region',
+        select: {
+          name: 1
+        }
+      }]
+    })
+  })
+})

test/generateElasticProjections.spec.js

@@ -0,0 +1,83 @@
+const expect = require('chai').expect
+const generateElasticProjections = require('../src/elasticsearch').generateElasticProjections
+
+const schemas = {
+  person: {
+    class: 'collection',
+    fields: {
+      name: {
+        type: 'string',
+        required: true,
+        es_indexed: true,
+        es_keyword: false
+      },
+      contact: {
+        type: 'reference',
+        ref: 'contact',
+        required: true,
+        es_indexed: true
+      },
+      address: {
+        type: 'reference',
+        ref: 'address',
+        required: false,
+        es_indexed: true
+      },
+      spouse: {
+        type: 'reference',
+        ref: 'person',
+        required: false,
+        es_indexed: true
+      },
+      aliases: {
+        type: 'array',
+        required: false,
+        item: {
+          type: 'reference',
+          required: false,
+          ref: 'person',
+          es_indexed: true
+        }
+      }
+    }
+  },
+  address: {
+    class: 'collection',
+    fields: {
+      line_1: {
+        type: 'string',
+        required: true,
+        es_indexed: true,
+        es_keyword: true
+      }
+    }
+  },
+  contact: {
+    class: 'embedded',
+    fields: {
+      email: {
+        type: 'string',
+        required: true,
+        es_indexed: true,
+        es_keyword: true
+      }
+    }
+  }
+}
+
+describe('generateElasticProjections', () => {
+  it('should generate all collection documents, projecting all fields except self referential fields', () => {
+    let projections = generateElasticProjections(schemas)
+
+    expect(projections).to.deep.equal({
+      person: {
+        name: 1,
+        contact: 1,
+        address: 1
+      },
+      address: {
+        line_1: 1
+      }
+    })
+  })
+})

test/manual/generateProjections.js

@@ -0,0 +1,6 @@
+const config = require('../data/configurationSchemas.js')
+const generateProjections = require('../../src/elasticsearch/generateProjections.js')
+
+let projections = generateProjections(config)
+
+console.log(projections)