Skip to content

Commit

Permalink
feat: add analytics module to handle /analytics endpoints (#403)
Browse files Browse the repository at this point in the history 
* feat(AnalyticsRouter): move /analytics routes into own router

* feat(AnalyticsService): add getUsersCount function

* refactor(AnalyticsRouter): replace handler for /users endpoint

* feat(AnalyticsService): add getSubmissionsCount

* feat(AnalyticsCtl): add handler for /submissions endpoint

* refactor(AnalyticsRouter): replace handler for /submissions endpoint

* feat(FeatureManager): add stronger typings and export type from index

* feat(SubmissionModel): add static method findFormsWithSubsAbove

* feat(FormStatsModel): add static method aggregateFormCount

* feat(AnalyticsService): add getFormCount methods stats/subs collection

* feat(AnalyticsFty): add factory to generate getFormCount method

* refactor(AnalyticsRouter): replace handler for /forms endpoint

* refactor(Analytics): rename handler functions to be more concise

* feat(FormStatsModel): add missing formId key

* test(SubmissionModel): add tests for static findFormsWithSubsAbove fn

* test(FormStatsModel): add tests for static aggregateFormCount fn

* test(AnalyticsService): add tests for getFormCount* methods

Also remove form count related jasmine tests

* test(AnalyticsService): add tests for getUserCount

* test(AnalyticsService): add tests for getSubmissionCount

* test(CoreCtl): remove tests for functions already refactored away

* test(AnalyticsFty): add tests

* test(AnalyticsCtl): add tests

* fix(AnalyticsService): correct logged meta action names

* test(AnalyticsCtl): tighten res.status call checks

* feat: add private JSDoc key to factory invoked methods
  • Loading branch information
@karrui
karrui committed Oct 6, 2020
1 parent 17d1db9 commit f7dfa49
Show file tree
Hide file tree
Showing 19 changed files with 1,110 additions and 389 deletions.
156 changes: 0 additions & 156 deletions src/app/controllers/core.server.controller.js
View file
Original file line number Diff line number Diff line change
@@ -1,21 +1,3 @@
const mongoose = require('mongoose')
const _ = require('lodash')
const { StatusCodes } = require('http-status-codes')

const config = require('../../config/config')
const { getRequestIp, getTrace } = require('../utils/request')
const logger = require('../../config/logger').createLoggerWithLabel(module)

const getFormStatisticsTotalModel = require('../models/form_statistics_total.server.model')
.default
const getSubmissionModel = require('../models/submission.server.model').default
const getUserModel = require('../models/user.server.model').default

const FormStatisticsTotal = getFormStatisticsTotalModel(mongoose)
const Submission = getSubmissionModel(mongoose)

const MIN_SUB_COUNT = 10 // minimum number of submissions before search is returned

/**
* Renders root: '/'
* @param {Object} req - Express request object
Expand All @@ -26,141 +8,3 @@ exports.index = function (req, res) {
user: JSON.stringify(req.session.user) || 'null',
})
}

/**
* Returns # forms that have > 10 responses using aggregate collection
* @param {Object} req - Express request object
* @param {Object} res - Express response object
*/
exports.formCountUsingAggregateCollection = (req, res) => {
FormStatisticsTotal.aggregate(
[
{
$match: {
totalCount: {
$gt: MIN_SUB_COUNT,
},
},
},
{
$count: 'numActiveForms',
},
],
function (err, [result]) {
if (err) {
logger.error({
message: 'Mongo form statistics aggregate error',
meta: {
action: 'formCountUsingAggregateCollection',
ip: getRequestIp(req),
trace: getTrace(req),
url: req.url,
headers: req.headers,
},
error: err,
})
res.sendStatus(StatusCodes.SERVICE_UNAVAILABLE)
} else if (result) {
res.json(_.get(result, 'numActiveForms', 0))
} else {
res.sendStatus(StatusCodes.INTERNAL_SERVER_ERROR)
}
},
)
}

/**
* Returns # forms that have > 10 responses using submissions collection
* @param {Object} req - Express request object
* @param {Object} res - Express response object
*/
exports.formCountUsingSubmissionsCollection = (req, res) => {
Submission.aggregate(
[
{
$group: {
_id: '$form',
count: { $sum: 1 },
},
},
{
$match: {
count: {
$gt: MIN_SUB_COUNT,
},
},
},
],
function (err, forms) {
if (err) {
logger.error({
message: 'Mongo submission aggregate error',
meta: {
action: 'formCountUsingSubmissionsCollection',
ip: getRequestIp(req),
trace: getTrace(req),
url: req.url,
headers: req.headers,
},
error: err,
})
res.sendStatus(StatusCodes.SERVICE_UNAVAILABLE)
} else {
res.json(forms.length)
}
},
)
}

/**
* Returns total number of users
* @param {Object} req - Express request object
* @param {Object} res - Express response object
*/
exports.userCount = (req, res) => {
let User = getUserModel(mongoose)
User.estimatedDocumentCount(function (err, ct) {
if (err) {
logger.error({
message: 'Mongo user count error',
meta: {
action: 'userCount',
ip: getRequestIp(req),
trace: getTrace(req),
url: req.url,
headers: req.headers,
},
error: err,
})
} else {
res.json(ct)
}
})
}

/**
* Returns total number of form submissions
* @param {Object} req - Express request object
* @param {Object} res - Express response object
*/
exports.submissionCount = (req, res) => {
let Submission = getSubmissionModel(mongoose)
Submission.estimatedDocumentCount(function (err, ct) {
if (err) {
logger.error({
message: 'Mongo submission count error',
meta: {
action: 'submissionCount',
ip: getRequestIp(req),
trace: getTrace(req),
url: req.url,
headers: req.headers,
},
error: err,
})
} else {
let totalCount = ct + config.submissionsTopUp
res.json(totalCount)
}
})
}
3 changes: 0 additions & 3 deletions src/app/factories/aggregate-stats.factory.js
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
const featureManager = require('../../config/feature-manager').default
const core = require('../controllers/core.server.controller')
const adminConsole = require('../controllers/admin-console.server.controller')

const aggregStatsFactory = ({ isEnabled }) => {
Expand All @@ -8,14 +7,12 @@ const aggregStatsFactory = ({ isEnabled }) => {
getExampleForms: adminConsole.getExampleFormsUsingAggregateCollection,
getSingleExampleForm:
adminConsole.getSingleExampleFormUsingAggregateCollection,
formCount: core.formCountUsingAggregateCollection,
}
} else {
return {
getExampleForms: adminConsole.getExampleFormsUsingSubmissionsCollection,
getSingleExampleForm:
adminConsole.getSingleExampleFormUsingSubmissionCollection,
formCount: core.formCountUsingSubmissionsCollection,
}
}
}
Expand Down
40 changes: 35 additions & 5 deletions src/app/models/form_statistics_total.server.model.ts
View file
Original file line number Diff line number Diff line change
@@ -1,15 +1,24 @@
import { Model, Mongoose, Schema } from 'mongoose'
import { Mongoose, Schema } from 'mongoose'

import { IFormStatisticsTotalSchema } from '../../types'
import {
AggregateFormCountResult,
IFormStatisticsTotalModel,
IFormStatisticsTotalSchema,
} from '../../types'

import { FORM_SCHEMA_ID } from './form.server.model'

const FORM_STATS_TOTAL_SCHEMA_ID = 'FormStatisticsTotal'
const FORM_STATS_COLLECTION_NAME = 'formStatisticsTotal'

type IFormStatisticsTotalModel = Model<IFormStatisticsTotalSchema>

const compileFormStatisticsTotalModel = (db: Mongoose) => {
const FormStatisticsTotalSchema = new Schema<IFormStatisticsTotalSchema>(
{
formId: {
type: Schema.Types.ObjectId,
ref: FORM_SCHEMA_ID,
required: true,
},
totalCount: {
type: Number,
required: true,
Expand Down Expand Up @@ -37,6 +46,25 @@ const compileFormStatisticsTotalModel = (db: Mongoose) => {
},
)

// Static functions
FormStatisticsTotalSchema.statics.aggregateFormCount = function (
this: IFormStatisticsTotalModel,
minSubCount: number,
): Promise<AggregateFormCountResult> {
return this.aggregate([
{
$match: {
totalCount: {
$gt: minSubCount,
},
},
},
{
$count: 'numActiveForms',
},
]).exec()
}

const FormStatisticsTotalModel = db.model<
IFormStatisticsTotalSchema,
IFormStatisticsTotalModel
Expand All @@ -55,7 +83,9 @@ const compileFormStatisticsTotalModel = (db: Mongoose) => {
* @param db The mongoose instance to retrieve the FormStatisticsTotal model from
* @returns The FormStatisticsTotal model
*/
const getFormStatisticsTotalModel = (db: Mongoose) => {
const getFormStatisticsTotalModel = (
db: Mongoose,
): IFormStatisticsTotalModel => {
try {
return db.model(FORM_STATS_TOTAL_SCHEMA_ID) as IFormStatisticsTotalModel
} catch {
Expand Down
58 changes: 44 additions & 14 deletions src/app/models/submission.server.model.ts
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,13 @@
import { Model, Mongoose, Schema } from 'mongoose'
import { Mongoose, Schema } from 'mongoose'

import {
AuthType,
FindFormsWithSubsAboveResult,
IEmailSubmissionModel,
IEmailSubmissionSchema,
IEncryptedSubmissionSchema,
IEncryptSubmissionModel,
ISubmissionModel,
ISubmissionSchema,
IWebhookResponseSchema,
MyInfoAttribute,
Expand Down Expand Up @@ -56,9 +58,29 @@ SubmissionSchema.index({
created: -1,
})

// Instance methods
// Base schema static methods
SubmissionSchema.statics.findFormsWithSubsAbove = function (
this: ISubmissionModel,
minSubCount: number,
): Promise<FindFormsWithSubsAboveResult[]> {
return this.aggregate<FindFormsWithSubsAboveResult>([
{
$group: {
_id: '$form',
count: { $sum: 1 },
},
},
{
$match: {
count: {
$gt: minSubCount,
},
},
},
]).exec()
}

const emailSubmissionSchema = new Schema<IEmailSubmissionSchema>({
const EmailSubmissionSchema = new Schema<IEmailSubmissionSchema>({
recipientEmails: {
type: [
{
Expand All @@ -83,10 +105,11 @@ const emailSubmissionSchema = new Schema<IEmailSubmissionSchema>({
},
})

// EmailSubmission Instance methods
/**
* Returns null as email submission does not have a webhook view
*/
emailSubmissionSchema.methods.getWebhookView = function (): null {
EmailSubmissionSchema.methods.getWebhookView = function (): null {
return null
}

Expand All @@ -110,7 +133,7 @@ const webhookResponseSchema = new Schema<IWebhookResponseSchema>(
},
)

const encryptSubmissionSchema = new Schema<IEncryptedSubmissionSchema>({
const EncryptSubmissionSchema = new Schema<IEncryptedSubmissionSchema>({
encryptedContent: {
type: String,
trim: true,
Expand All @@ -135,7 +158,7 @@ const encryptSubmissionSchema = new Schema<IEncryptedSubmissionSchema>({
* Returns an object which represents the encrypted submission
* which will be posted to the webhook URL.
*/
encryptSubmissionSchema.methods.getWebhookView = function (
EncryptSubmissionSchema.methods.getWebhookView = function (
this: IEncryptedSubmissionSchema,
): WebhookView {
const webhookData: WebhookData = {
Expand All @@ -152,27 +175,34 @@ encryptSubmissionSchema.methods.getWebhookView = function (
}
}

const compileSubmissionModel = (db: Mongoose) => {
const compileSubmissionModel = (db: Mongoose): ISubmissionModel => {
const Submission = db.model('Submission', SubmissionSchema)
Submission.discriminator(SubmissionType.Email, emailSubmissionSchema)
Submission.discriminator(SubmissionType.Encrypt, encryptSubmissionSchema)
return db.model<ISubmissionSchema>(SUBMISSION_SCHEMA_ID, SubmissionSchema)
Submission.discriminator(SubmissionType.Email, EmailSubmissionSchema)
Submission.discriminator(SubmissionType.Encrypt, EncryptSubmissionSchema)
return db.model<ISubmissionSchema>(
SUBMISSION_SCHEMA_ID,
SubmissionSchema,
) as ISubmissionModel
}

const getSubmissionModel = (db: Mongoose) => {
const getSubmissionModel = (db: Mongoose): ISubmissionModel => {
try {
return db.model(SUBMISSION_SCHEMA_ID) as Model<ISubmissionSchema>
return db.model(SUBMISSION_SCHEMA_ID) as ISubmissionModel
} catch {
return compileSubmissionModel(db)
}
}

export const getEmailSubmissionModel = (db: Mongoose) => {
export const getEmailSubmissionModel = (
db: Mongoose,
): IEmailSubmissionModel => {
getSubmissionModel(db)
return db.model(SubmissionType.Email) as IEmailSubmissionModel
}

export const getEncryptSubmissionModel = (db: Mongoose) => {
export const getEncryptSubmissionModel = (
db: Mongoose,
): IEncryptSubmissionModel => {
getSubmissionModel(db)
return db.model(SubmissionType.Encrypt) as IEncryptSubmissionModel
}
Expand Down
Loading

0 comments on commit f7dfa49

Please sign in to comment.