From cc703ccbfb48921448ce9941a0e04d4b1848977c Mon Sep 17 00:00:00 2001 From: anilb Date: Wed, 1 Feb 2023 16:15:00 +0100 Subject: [PATCH 01/36] eagle eye content v2 start --- .../U1675259471__eagleEyeActions.sql | 59 ++++++++++++++++++ .../V1675259471__eagleEyeActions.sql | 32 ++++++++++ backend/src/database/models/eagleEyeAction.ts | 62 +++++++++++++++++++ .../src/database/models/eagleEyeContent.ts | 36 +++++------ 4 files changed, 170 insertions(+), 19 deletions(-) create mode 100644 backend/src/database/migrations/U1675259471__eagleEyeActions.sql create mode 100644 backend/src/database/migrations/V1675259471__eagleEyeActions.sql create mode 100644 backend/src/database/models/eagleEyeAction.ts diff --git a/backend/src/database/migrations/U1675259471__eagleEyeActions.sql b/backend/src/database/migrations/U1675259471__eagleEyeActions.sql new file mode 100644 index 0000000000..30b8b2c037 --- /dev/null +++ b/backend/src/database/migrations/U1675259471__eagleEyeActions.sql @@ -0,0 +1,59 @@ +DROP TABLE IF EXISTS "eagleEyeContents"; + +DROP TYPE "eagleEyeContents_actions_type"; + +create table "eagleEyeContents"; +( + id uuid not null primary key, + "sourceId" text not null, + "vectorId" text not null, + status varchar(255) default NULL::character varying, + title text not null, + username text not null, + url text not null, + text text, + timestamp timestamp with time zone not null, + platform text not null, + keywords text [], + "similarityScore" double precision, + "userAttributes" jsonb, + "postAttributes" jsonb, + "importHash" varchar(255), + "createdAt" timestamp with time zone not null, + "updatedAt" timestamp with time zone not null, + "deletedAt" timestamp with time zone, + "tenantId" uuid not null references tenants on update cascade, + "createdById" uuid references users on update cascade on delete + set null, + "updatedById" uuid references users on update cascade on delete + set null, + "exactKeywords" text [] +); + +alter table "eagleEyeContents" owner to postgres; + +create index discord on "eagleEyeContents" ("vectorId", status); + +create index members_email_tenant_id on "eagleEyeContents" (id) +where ("deletedAt" IS NULL); + +create index members_joined_at_tenant_id on "eagleEyeContents" (id) +where ("deletedAt" IS NULL); + +create index members_username on "eagleEyeContents" using gin (id); + +create index slack on "eagleEyeContents" (id); + +create index twitter on "eagleEyeContents" (id); + +create unique index eagle_eye_contents_import_hash_tenant_id on "eagleEyeContents" ("importHash", "tenantId") +where ("deletedAt" IS NULL); + +create index eagle_eye_contents_platform_tenant_id_timestamp on "eagleEyeContents" (platform, "tenantId", timestamp) +where ("deletedAt" IS NULL); + +create index eagle_eye_contents_status_tenant_id_timestamp on "eagleEyeContents" (status, "tenantId", timestamp) +where ("deletedAt" IS NULL); + +create index eagle_eye_contents_tenant_id_timestamp on "eagleEyeContents" ("tenantId", timestamp) +where ("deletedAt" IS NULL); \ No newline at end of file diff --git a/backend/src/database/migrations/V1675259471__eagleEyeActions.sql b/backend/src/database/migrations/V1675259471__eagleEyeActions.sql new file mode 100644 index 0000000000..6ca96a3afb --- /dev/null +++ b/backend/src/database/migrations/V1675259471__eagleEyeActions.sql @@ -0,0 +1,32 @@ +DROP TABLE IF EXISTS "eagleEyeContents"; + +CREATE TABLE public."eageEyeContents" ( + "id" uuid NOT NULL, + "content" jsonb NOT NULL, + "tenantId" uuid NOT NULL, + "createdAt" timestamptz NOT NULL, + "updatedAt" timestamptz NOT NULL, + CONSTRAINT "eageEyeContents_pkey" PRIMARY KEY ("id") +); + +ALTER TABLE public."eageEyeContents" ADD CONSTRAINT "eageEyeContents_tenantId_fkey" FOREIGN KEY ("tenantId") REFERENCES public.tenants(id) ON DELETE NO ACTION ON UPDATE NO ACTION; + +CREATE TYPE "eagleEyeActions_action_type" AS ENUM ('thumbs-up', 'thumbs-down', 'bookmark'); + +CREATE TABLE public."eageEyeActions" ( + "id" uuid NOT NULL, + "platform" text NOT NULL, + "action" public."eagleEyeActions_actions_type" NOT NULL, + "timestamp" timestamptz NOT NULL, + "url" text NOT NULL, + "contentId" uuid NOT NULL, + "tenantId" uuid NOT NULL, + "actionBy" uuid NOT NULL, + "createdAt" timestamptz NOT NULL, + "updatedAt" timestamptz NOT NULL, + CONSTRAINT "eageEyeActions_pkey" PRIMARY KEY ("id") +); + +ALTER TABLE public."eageEyeActions" ADD CONSTRAINT "eageEyeActions_tenantId_fkey" FOREIGN KEY ("tenantId") REFERENCES public.tenants(id) ON DELETE NO ACTION ON UPDATE NO ACTION; +ALTER TABLE public."eageEyeActions" ADD CONSTRAINT "eageEyeActions_actionBy_fkey" FOREIGN KEY ("actionBy") REFERENCES public.users(id) ON DELETE NO ACTION ON UPDATE NO ACTION; +ALTER TABLE public."eageEyeActions" ADD CONSTRAINT "eageEyeActions_contentId_fkey" FOREIGN KEY ("contentId") REFERENCES public."eageEyeContents"(id) ON DELETE NO ACTION ON UPDATE NO ACTION; diff --git a/backend/src/database/models/eagleEyeAction.ts b/backend/src/database/models/eagleEyeAction.ts new file mode 100644 index 0000000000..305cef39e9 --- /dev/null +++ b/backend/src/database/models/eagleEyeAction.ts @@ -0,0 +1,62 @@ +import { DataTypes } from 'sequelize' + +const eagleEyeActionModel = { + id: { + type: DataTypes.UUID, + defaultValue: DataTypes.UUIDV4, + primaryKey: true, + }, + platform: { + type: DataTypes.TEXT, + allowNull: false, + validate: { + notEmpty: true, + }, + }, + action: { + type: DataTypes.TEXT, + validate: { + isIn: [['thumbs-up', 'thumbs-down', 'bookmark']], + }, + defaultValue: null, + }, + timestamp: { + type: DataTypes.DATE, + allowNull: false, + }, + url: { + type: DataTypes.TEXT, + allowNull: false, + validate: { + notEmpty: true, + }, + } +} + +export default (sequelize) => { + const eagleEyeAction = sequelize.define('eagleEyeAction', eagleEyeActionModel, { + timestamps: true, + paranoid: false, + }) + + eagleEyeAction.associate = (models) => { + models.eagleEyeContent.belongsTo(models.tenant, { + as: 'tenant', + foreignKey: { + allowNull: false, + }, + }) + + models.eagleEyeAction.belongsTo(models.user, { + as: 'actionBy', + }) + + models.eagleEyeAction.belongsTo(models.eagleEyeContent, { + as: 'content', + }) + } + + return eagleEyeAction +} + +export { eagleEyeActionModel } diff --git a/backend/src/database/models/eagleEyeContent.ts b/backend/src/database/models/eagleEyeContent.ts index bfee48a92e..d827d7972d 100644 --- a/backend/src/database/models/eagleEyeContent.ts +++ b/backend/src/database/models/eagleEyeContent.ts @@ -6,26 +6,31 @@ const eagleEyeContentModel = { defaultValue: DataTypes.UUIDV4, primaryKey: true, }, - sourceId: { + post: { + type: DataTypes.JSONB, + allowNull: false + }, + action: { type: DataTypes.TEXT, - allowNull: false, validate: { - notEmpty: true, + isIn: [['thumbs-up', 'thumbs-down', 'bookmark']], }, + defaultValue: null, + }, + timestamp: { + type: DataTypes.DATE, + allowNull: false, }, - vectorId: { + url: { type: DataTypes.TEXT, allowNull: false, validate: { notEmpty: true, }, }, - status: { - type: DataTypes.STRING(255), - validate: { - isIn: [['engaged', 'rejected']], - }, - defaultValue: null, + post: { + type: DataTypes.JSONB, + allowNull: false }, title: { type: DataTypes.TEXT, @@ -51,10 +56,7 @@ const eagleEyeContentModel = { text: { type: DataTypes.TEXT, }, - timestamp: { - type: DataTypes.DATE, - allowNull: false, - }, + platform: { type: DataTypes.TEXT, allowNull: false, @@ -75,7 +77,6 @@ const eagleEyeContentModel = { }, userAttributes: { type: DataTypes.JSONB, - default: {}, }, postAttributes: { type: DataTypes.JSONB, @@ -132,12 +133,9 @@ export default (sequelize) => { }) models.eagleEyeContent.belongsTo(models.user, { - as: 'createdBy', + as: 'actionBy', }) - models.eagleEyeContent.belongsTo(models.user, { - as: 'updatedBy', - }) } return eagleEyeContent From 12b766f11672222769027f62c5575146f9c9086e Mon Sep 17 00:00:00 2001 From: anil Date: Thu, 2 Feb 2023 10:33:10 +0100 Subject: [PATCH 02/36] schema updates --- .../V1675259471__eagleEyeActions.sql | 8 +- .../src/database/models/eagleEyeContent.ts | 108 +----------------- 2 files changed, 5 insertions(+), 111 deletions(-) diff --git a/backend/src/database/migrations/V1675259471__eagleEyeActions.sql b/backend/src/database/migrations/V1675259471__eagleEyeActions.sql index 6ca96a3afb..96303250a4 100644 --- a/backend/src/database/migrations/V1675259471__eagleEyeActions.sql +++ b/backend/src/database/migrations/V1675259471__eagleEyeActions.sql @@ -11,22 +11,22 @@ CREATE TABLE public."eageEyeContents" ( ALTER TABLE public."eageEyeContents" ADD CONSTRAINT "eageEyeContents_tenantId_fkey" FOREIGN KEY ("tenantId") REFERENCES public.tenants(id) ON DELETE NO ACTION ON UPDATE NO ACTION; -CREATE TYPE "eagleEyeActions_action_type" AS ENUM ('thumbs-up', 'thumbs-down', 'bookmark'); +CREATE TYPE public."eagleEyeActions_action_type" AS ENUM ('thumbs-up', 'thumbs-down', 'bookmark'); CREATE TABLE public."eageEyeActions" ( "id" uuid NOT NULL, "platform" text NOT NULL, - "action" public."eagleEyeActions_actions_type" NOT NULL, + "action" public."eagleEyeActions_action_type" NOT NULL, "timestamp" timestamptz NOT NULL, "url" text NOT NULL, "contentId" uuid NOT NULL, "tenantId" uuid NOT NULL, - "actionBy" uuid NOT NULL, + "actionById" uuid NOT NULL, "createdAt" timestamptz NOT NULL, "updatedAt" timestamptz NOT NULL, CONSTRAINT "eageEyeActions_pkey" PRIMARY KEY ("id") ); ALTER TABLE public."eageEyeActions" ADD CONSTRAINT "eageEyeActions_tenantId_fkey" FOREIGN KEY ("tenantId") REFERENCES public.tenants(id) ON DELETE NO ACTION ON UPDATE NO ACTION; -ALTER TABLE public."eageEyeActions" ADD CONSTRAINT "eageEyeActions_actionBy_fkey" FOREIGN KEY ("actionBy") REFERENCES public.users(id) ON DELETE NO ACTION ON UPDATE NO ACTION; +ALTER TABLE public."eageEyeActions" ADD CONSTRAINT "eageEyeActions_actionBy_fkey" FOREIGN KEY ("actionById") REFERENCES public.users(id) ON DELETE NO ACTION ON UPDATE NO ACTION; ALTER TABLE public."eageEyeActions" ADD CONSTRAINT "eageEyeActions_contentId_fkey" FOREIGN KEY ("contentId") REFERENCES public."eageEyeContents"(id) ON DELETE NO ACTION ON UPDATE NO ACTION; diff --git a/backend/src/database/models/eagleEyeContent.ts b/backend/src/database/models/eagleEyeContent.ts index d827d7972d..f7cfae3e59 100644 --- a/backend/src/database/models/eagleEyeContent.ts +++ b/backend/src/database/models/eagleEyeContent.ts @@ -6,120 +6,14 @@ const eagleEyeContentModel = { defaultValue: DataTypes.UUIDV4, primaryKey: true, }, - post: { + content: { type: DataTypes.JSONB, allowNull: false }, - action: { - type: DataTypes.TEXT, - validate: { - isIn: [['thumbs-up', 'thumbs-down', 'bookmark']], - }, - defaultValue: null, - }, - timestamp: { - type: DataTypes.DATE, - allowNull: false, - }, - url: { - type: DataTypes.TEXT, - allowNull: false, - validate: { - notEmpty: true, - }, - }, - post: { - type: DataTypes.JSONB, - allowNull: false - }, - title: { - type: DataTypes.TEXT, - allowNull: false, - validate: { - notEmpty: true, - }, - }, - username: { - type: DataTypes.TEXT, - allowNull: false, - validate: { - notEmpty: true, - }, - }, - url: { - type: DataTypes.TEXT, - allowNull: false, - validate: { - notEmpty: true, - }, - }, - text: { - type: DataTypes.TEXT, - }, - - platform: { - type: DataTypes.TEXT, - allowNull: false, - validate: { - notEmpty: true, - }, - }, - keywords: { - type: DataTypes.ARRAY(DataTypes.TEXT), - default: [], - }, - exactKeywords: { - type: DataTypes.ARRAY(DataTypes.TEXT), - default: [], - }, - similarityScore: { - type: DataTypes.FLOAT, - }, - userAttributes: { - type: DataTypes.JSONB, - }, - postAttributes: { - type: DataTypes.JSONB, - default: {}, - }, - importHash: { - type: DataTypes.STRING(255), - allowNull: true, - validate: { - len: [0, 255], - }, - }, } export default (sequelize) => { const eagleEyeContent = sequelize.define('eagleEyeContent', eagleEyeContentModel, { - indexes: [ - { - unique: true, - fields: ['importHash', 'tenantId'], - where: { - deletedAt: null, - }, - }, - { - fields: ['platform', 'tenantId', 'timestamp'], - where: { - deletedAt: null, - }, - }, - { - fields: ['status', 'tenantId', 'timestamp'], - where: { - deletedAt: null, - }, - }, - { - fields: ['tenantId', 'timestamp'], - where: { - deletedAt: null, - }, - }, - ], timestamps: true, paranoid: true, }) From 79266bd30155c5d306b88a9f3a1b4acf73a0019c Mon Sep 17 00:00:00 2001 From: anilb Date: Thu, 2 Feb 2023 12:37:08 +0100 Subject: [PATCH 03/36] schema updates, actions repo start --- .../V1675259471__eagleEyeActions.sql | 4 ++-- backend/src/database/models/eagleEyeAction.ts | 14 -------------- backend/src/database/models/eagleEyeContent.ts | 18 ++++++++++++++---- .../repositories/eagleEyeActionRepository.ts | 7 +++++++ 4 files changed, 23 insertions(+), 20 deletions(-) create mode 100644 backend/src/database/repositories/eagleEyeActionRepository.ts diff --git a/backend/src/database/migrations/V1675259471__eagleEyeActions.sql b/backend/src/database/migrations/V1675259471__eagleEyeActions.sql index 96303250a4..042f8d6cf2 100644 --- a/backend/src/database/migrations/V1675259471__eagleEyeActions.sql +++ b/backend/src/database/migrations/V1675259471__eagleEyeActions.sql @@ -2,6 +2,8 @@ DROP TABLE IF EXISTS "eagleEyeContents"; CREATE TABLE public."eageEyeContents" ( "id" uuid NOT NULL, + "platform" text NOT NULL, + "url" text NOT NULL, "content" jsonb NOT NULL, "tenantId" uuid NOT NULL, "createdAt" timestamptz NOT NULL, @@ -15,10 +17,8 @@ CREATE TYPE public."eagleEyeActions_action_type" AS ENUM ('thumbs-up', 'thumbs-d CREATE TABLE public."eageEyeActions" ( "id" uuid NOT NULL, - "platform" text NOT NULL, "action" public."eagleEyeActions_action_type" NOT NULL, "timestamp" timestamptz NOT NULL, - "url" text NOT NULL, "contentId" uuid NOT NULL, "tenantId" uuid NOT NULL, "actionById" uuid NOT NULL, diff --git a/backend/src/database/models/eagleEyeAction.ts b/backend/src/database/models/eagleEyeAction.ts index 305cef39e9..b47d8126b0 100644 --- a/backend/src/database/models/eagleEyeAction.ts +++ b/backend/src/database/models/eagleEyeAction.ts @@ -6,13 +6,6 @@ const eagleEyeActionModel = { defaultValue: DataTypes.UUIDV4, primaryKey: true, }, - platform: { - type: DataTypes.TEXT, - allowNull: false, - validate: { - notEmpty: true, - }, - }, action: { type: DataTypes.TEXT, validate: { @@ -24,13 +17,6 @@ const eagleEyeActionModel = { type: DataTypes.DATE, allowNull: false, }, - url: { - type: DataTypes.TEXT, - allowNull: false, - validate: { - notEmpty: true, - }, - } } export default (sequelize) => { diff --git a/backend/src/database/models/eagleEyeContent.ts b/backend/src/database/models/eagleEyeContent.ts index f7cfae3e59..8e7e7ab702 100644 --- a/backend/src/database/models/eagleEyeContent.ts +++ b/backend/src/database/models/eagleEyeContent.ts @@ -6,10 +6,24 @@ const eagleEyeContentModel = { defaultValue: DataTypes.UUIDV4, primaryKey: true, }, + platform: { + type: DataTypes.TEXT, + allowNull: false, + validate: { + notEmpty: true, + }, + }, content: { type: DataTypes.JSONB, allowNull: false }, + url: { + type: DataTypes.TEXT, + allowNull: false, + validate: { + notEmpty: true, + }, + } } export default (sequelize) => { @@ -26,10 +40,6 @@ export default (sequelize) => { }, }) - models.eagleEyeContent.belongsTo(models.user, { - as: 'actionBy', - }) - } return eagleEyeContent diff --git a/backend/src/database/repositories/eagleEyeActionRepository.ts b/backend/src/database/repositories/eagleEyeActionRepository.ts new file mode 100644 index 0000000000..8fb48cce8b --- /dev/null +++ b/backend/src/database/repositories/eagleEyeActionRepository.ts @@ -0,0 +1,7 @@ +import { IRepositoryOptions } from './IRepositoryOptions' + +export default class EagleEyeActionRepository { + + + +} \ No newline at end of file From 47e75a8bacf45528d658982676d1cea9037f1fc1 Mon Sep 17 00:00:00 2001 From: anilb Date: Thu, 2 Feb 2023 18:18:50 +0100 Subject: [PATCH 04/36] repo layer progress --- .../V1675259471__eagleEyeActions.sql | 2 +- .../src/database/models/eagleEyeContent.ts | 2 +- .../repositories/eagleEyeContentRepository.ts | 367 +++--------------- backend/src/types/eagleEyeTypes.ts | 24 ++ 4 files changed, 84 insertions(+), 311 deletions(-) create mode 100644 backend/src/types/eagleEyeTypes.ts diff --git a/backend/src/database/migrations/V1675259471__eagleEyeActions.sql b/backend/src/database/migrations/V1675259471__eagleEyeActions.sql index 042f8d6cf2..6ec06fe385 100644 --- a/backend/src/database/migrations/V1675259471__eagleEyeActions.sql +++ b/backend/src/database/migrations/V1675259471__eagleEyeActions.sql @@ -4,7 +4,7 @@ CREATE TABLE public."eageEyeContents" ( "id" uuid NOT NULL, "platform" text NOT NULL, "url" text NOT NULL, - "content" jsonb NOT NULL, + "post" jsonb NOT NULL, "tenantId" uuid NOT NULL, "createdAt" timestamptz NOT NULL, "updatedAt" timestamptz NOT NULL, diff --git a/backend/src/database/models/eagleEyeContent.ts b/backend/src/database/models/eagleEyeContent.ts index 8e7e7ab702..8b11be5333 100644 --- a/backend/src/database/models/eagleEyeContent.ts +++ b/backend/src/database/models/eagleEyeContent.ts @@ -13,7 +13,7 @@ const eagleEyeContentModel = { notEmpty: true, }, }, - content: { + post: { type: DataTypes.JSONB, allowNull: false }, diff --git a/backend/src/database/repositories/eagleEyeContentRepository.ts b/backend/src/database/repositories/eagleEyeContentRepository.ts index 56de3e60f0..ca7aff5eef 100644 --- a/backend/src/database/repositories/eagleEyeContentRepository.ts +++ b/backend/src/database/repositories/eagleEyeContentRepository.ts @@ -5,8 +5,7 @@ import Error404 from '../../errors/Error404' import Error400 from '../../errors/Error400' import AuditLogRepository from './auditLogRepository' import { IRepositoryOptions } from './IRepositoryOptions' -import QueryParser from './filters/queryParser' -import { QueryOutput } from './filters/queryTypes' +import { EagleEyeContentData } from '../../types/eagleEyeTypes' export default class EagleEyeContentRepository { /** @@ -15,300 +14,49 @@ export default class EagleEyeContentRepository { * @param options Repository options. * @returns Created EagleEyeContent record. */ - static async upsert(data, options: IRepositoryOptions) { - const currentUser = SequelizeRepository.getCurrentUser(options) - - const tenant = SequelizeRepository.getCurrentTenant(options) - - const transaction = SequelizeRepository.getTransaction(options) + static async upsert(data:EagleEyeContentData, options: IRepositoryOptions): Promise { - if (data.status && ![null, 'rejected', 'engaged'].includes(data.status)) { - throw new Error400('en', 'errors.invalidEagleEyeStatus.message') + if(!data.url){ + throw new Error(`Can't upsert without url`) } - const existing = await options.database.eagleEyeContent.findOne({ - where: { - tenantId: tenant.id, - sourceId: data.sourceId, - }, - }) - - // If the content is already shown, we don't need to add it again - if (existing) { - // If the content comes from a different kewword, we also add it - if (!lodash.isEqual(data.keywords.sort(), existing.keywords.sort())) { - const keywords = lodash.uniq([...existing.keywords, ...data.keywords]) - let exactKeywords = null - if (data.exactKeywords && !existing.exactKeywords) { - exactKeywords = data.exactKeywords - } else if (!data.exactKeywords && existing.exactKeywords) { - exactKeywords = existing.exactKeywords - } - if (data.exactKeywords && existing.exactKeywords) { - exactKeywords = lodash.uniq([...existing.exactKeywords, ...data.exactKeywords]) - } - const similarityScore = data.similarityScore - return existing.update( - { - keywords, - similarityScore, - exactKeywords, - }, - { - transaction, - }, - ) - } - return existing - } + // find by url + const existing = await EagleEyeContentRepository.findByUrl(data.url, options) - if (typeof data.keywords === 'string') { - data.keywords = [data.keywords] - } + let record - if (typeof data.exactKeywords === 'string') { - data.exactKeywords = [data.exactKeywords] + if (existing){ + record = await EagleEyeContentRepository.update(existing.id, data, options) } - - if (typeof data.timestamp === 'number') { - data.timestamp = moment.unix(data.timestamp).toDate() + /* + else{ + record = options.database.eagleEyeContent.create( + { + ...lodash.pick(data, [ + 'platform', + 'post', + 'url', + ]), + memberId: data.member || null, + parentId: data.parent || null, + sourceParentId: data.sourceParentId || null, + conversationId: data.conversationId || null, + tenantId: tenant.id, + createdById: currentUser.id, + updatedById: currentUser.id, + }, + { + transaction, + }, + ) } + */ - const record = await options.database.eagleEyeContent.create( - { - ...lodash.pick(data, [ - 'sourceId', - 'vectorId', - 'status', - 'postAttributes', - 'title', - 'username', - 'url', - 'text', - 'timestamp', - 'userAttributes', - 'platform', - 'keywords', - 'exactKeywords', - 'similarityScore', - 'importHash', - ]), - - tenantId: tenant.id, - createdById: currentUser.id, - updatedById: currentUser.id, - }, - { - transaction, - }, - ) - - await AuditLogRepository.log( - { - entityName: 'eagleEyeContent', - entityId: record.id, - action: AuditLogRepository.CREATE, - values: data, - }, - options, - ) + return this.findById(record.id, options) } - /** - * EagleEyeContent find all records matching given criteria. - * @returns Records found. - */ - static async findAndCountAll( - { filter = {} as any, advancedFilter = null as any, limit = 0, offset = 0, orderBy = '' }, - options: IRepositoryOptions, - ) { - // If the advanced filter is empty, we construct it from the query parameter filter - if (!advancedFilter) { - advancedFilter = { and: [] } - - if (filter.id) { - advancedFilter.and.push({ - id: filter.id, - }) - } - - if (filter.sourceId) { - advancedFilter.and.push({ - sourceId: filter.sourceId, - }) - } - - if (filter.vectorId) { - advancedFilter.and.push({ - vectorId: filter.vectorId, - }) - } - - if (filter.status) { - if (filter.status === 'NULL') { - advancedFilter.and.push({ - status: 'NULL', - }) - } else if (filter.status === 'NOT_NULL') { - advancedFilter.and.push({ - status: { - not: null, - }, - }) - } else { - advancedFilter.and.push({ - status: { - textContains: filter.status, - }, - }) - } - } - - if (filter.timestampRange) { - const [start, end] = filter.timestampRange - - if (start !== undefined && start !== null && start !== '') { - advancedFilter.and.push({ - timestamp: { - gte: start, - }, - }) - } - - if (end !== undefined && end !== null && end !== '') { - advancedFilter.and.push({ - timestamp: { - lte: end, - }, - }) - } - } - - if (filter.platforms) { - advancedFilter.and.push({ - platform: { - or: filter.platforms.split(','), - }, - }) - } - - if (filter.nDays) { - advancedFilter.and.push({ - timestamp: { - gte: moment().subtract(filter.nDays, 'days').toDate(), - }, - }) - } - - if (filter.title) { - advancedFilter.and.push({ - title: { - textContains: filter.title, - }, - }) - } - - if (filter.text) { - advancedFilter.and.push({ - text: { - textContains: filter.text, - }, - }) - } - - if (filter.url) { - advancedFilter.and.push({ - url: { - textContains: filter.url, - }, - }) - } - - if (filter.username) { - advancedFilter.and.push({ - username: { - textContains: filter.username, - }, - }) - } - - if (filter.keywords) { - // Overlap will take a post where any keyword matches any of the filter keywords - advancedFilter.and.push({ - keywords: { - overlap: filter.keywords.split(','), - }, - }) - } - - if (filter.similarityScoreRange) { - const [start, end] = filter.similarityScoreRange - - if (start !== undefined && start !== null && start !== '') { - advancedFilter.and.push({ - similarityScore: { - gte: start, - }, - }) - } - - if (end !== undefined && end !== null && end !== '') { - advancedFilter.and.push({ - similarityScore: { - lte: end, - }, - }) - } - } - - if (filter.createdAtRange) { - const [start, end] = filter.createdAtRange - - if (start !== undefined && start !== null && start !== '') { - advancedFilter.and.push({ - createdAt: { - gte: start, - }, - }) - } - - if (end !== undefined && end !== null && end !== '') { - advancedFilter.and.push({ - createdAt: { - lte: end, - }, - }) - } - } - } - - const parser = new QueryParser({}, options) - - const parsed: QueryOutput = parser.parse({ - filter: advancedFilter, - orderBy: orderBy || ['createdAt_DESC'], - limit, - offset, - }) - - let { - rows, - count, // eslint-disable-line prefer-const - } = await options.database.eagleEyeContent.findAndCountAll({ - ...(parsed.where ? { where: parsed.where } : {}), - ...(parsed.having ? { having: parsed.having } : {}), - order: parsed.order, - limit: parsed.limit, - offset: parsed.offset, - transaction: SequelizeRepository.getTransaction(options), - }) - - rows = await this._populateRelationsForRows(rows) - - return { rows, count, limit: parsed.limit, offset: parsed.offset } - } static async update(id, data, options: IRepositoryOptions) { const currentUser = SequelizeRepository.getCurrentUser(options) @@ -329,26 +77,13 @@ export default class EagleEyeContentRepository { throw new Error404() } - if (data.status && ![null, 'rejected', 'engaged'].includes(data.status)) { - throw new Error400('en', 'errors.invalidEagleEyeStatus.message') - } record = await record.update( { ...lodash.pick(data, [ - 'sourceId', - 'vectorId', - 'status', - 'title', - 'username', - 'url', - 'text', - 'postAttributes', - 'timestamp', 'platform', - 'userAttributes', - 'importHash', - // Missing keywords on purpose + 'post', + 'url', ]), updatedById: currentUser.id, }, @@ -357,19 +92,10 @@ export default class EagleEyeContentRepository { }, ) - await AuditLogRepository.log( - { - entityName: 'eagleEyeContent', - entityId: record.id, - action: AuditLogRepository.UPDATE, - values: data, - }, - options, - ) return this.findById(record.id, options) } - static async findById(id, options: IRepositoryOptions) { + static async findById(id:string, options: IRepositoryOptions) { const transaction = SequelizeRepository.getTransaction(options) const include = [] @@ -392,6 +118,29 @@ export default class EagleEyeContentRepository { return this._populateRelations(record) } + static async findByUrl(url:string, options: IRepositoryOptions) { + const transaction = SequelizeRepository.getTransaction(options) + + const include = [] + + const currentTenant = SequelizeRepository.getCurrentTenant(options) + + const record = await options.database.eagleEyeContent.findOne({ + where: { + url, + tenantId: currentTenant.id, + }, + include, + transaction, + }) + + if (!record) { + return null + } + + return this._populateRelations(record) + } + static async _populateRelationsForRows(rows) { if (!rows) { return rows diff --git a/backend/src/types/eagleEyeTypes.ts b/backend/src/types/eagleEyeTypes.ts new file mode 100644 index 0000000000..0ffbaaf1f2 --- /dev/null +++ b/backend/src/types/eagleEyeTypes.ts @@ -0,0 +1,24 @@ +export enum EagleEyeActionType { + THUMBS_UP = 'thumbs-up', + THUMBS_DOWN = 'thumbs-down', + BOOKMARK = 'bookmark', +} + +export interface EagleEyeContentData { + id?: string + platform: string + post: any + url: string + tenantId: string + createdAt?: string + updatedAt?: string +} + +export interface EagleEyeActionData { + id?: string + action: EagleEyeActionType + timestamp: string + content: EagleEyeContentData + createdAt?: string + updatedAt?: string +} From f7866611b741937d8b29ca64aa6ef2bdba7aa732 Mon Sep 17 00:00:00 2001 From: anil Date: Sun, 5 Feb 2023 01:58:29 +0100 Subject: [PATCH 05/36] filtering with tests, endpoint routing remaining --- .../eagleEyeContent/eagleEyeActionCreate.ts | 0 .../eagleEyeContent/eagleEyeContentUpsert.ts | 0 backend/src/api/eagleEyeContent/index.ts | 6 + .../V1675259471__eagleEyeActions.sql | 21 +- backend/src/database/models/eagleEyeAction.ts | 4 +- .../src/database/models/eagleEyeContent.ts | 12 +- backend/src/database/models/index.ts | 1 + .../eagleEyeActionRepository.test.ts | 61 +++ .../eagleEyeContentRepository.test.ts | 371 ++++++++---------- .../repositories/eagleEyeActionRepository.ts | 122 +++++- .../repositories/eagleEyeContentRepository.ts | 204 +++++++--- .../src/database/utils/sequelizeTestUtils.ts | 1 + .../__tests__/eagleEyeContentService.test.ts | 149 ++----- backend/src/services/eagleEyeActionService.ts | 79 ++++ .../src/services/eagleEyeContentService.ts | 178 ++------- backend/src/types/common.ts | 7 + backend/src/types/eagleEyeTypes.ts | 25 +- 17 files changed, 698 insertions(+), 543 deletions(-) create mode 100644 backend/src/api/eagleEyeContent/eagleEyeActionCreate.ts create mode 100644 backend/src/api/eagleEyeContent/eagleEyeContentUpsert.ts create mode 100644 backend/src/database/repositories/__tests__/eagleEyeActionRepository.test.ts create mode 100644 backend/src/services/eagleEyeActionService.ts diff --git a/backend/src/api/eagleEyeContent/eagleEyeActionCreate.ts b/backend/src/api/eagleEyeContent/eagleEyeActionCreate.ts new file mode 100644 index 0000000000..e69de29bb2 diff --git a/backend/src/api/eagleEyeContent/eagleEyeContentUpsert.ts b/backend/src/api/eagleEyeContent/eagleEyeContentUpsert.ts new file mode 100644 index 0000000000..e69de29bb2 diff --git a/backend/src/api/eagleEyeContent/index.ts b/backend/src/api/eagleEyeContent/index.ts index 33928e9148..5cd3220e19 100644 --- a/backend/src/api/eagleEyeContent/index.ts +++ b/backend/src/api/eagleEyeContent/index.ts @@ -18,4 +18,10 @@ export default (app) => { `/tenant/:tenantId/eagleEyeContent/:id`, safeWrap(require('./eagleEyeContentFind').default), ) + + app.post( + `/tenant/:tenantId/eagleEyeContent/:contentId/action`, + safeWrap(require('./eagleEyeActionCreate').default), + ) + } diff --git a/backend/src/database/migrations/V1675259471__eagleEyeActions.sql b/backend/src/database/migrations/V1675259471__eagleEyeActions.sql index 6ec06fe385..9183281bc1 100644 --- a/backend/src/database/migrations/V1675259471__eagleEyeActions.sql +++ b/backend/src/database/migrations/V1675259471__eagleEyeActions.sql @@ -1,32 +1,33 @@ DROP TABLE IF EXISTS "eagleEyeContents"; -CREATE TABLE public."eageEyeContents" ( +CREATE TABLE public."eagleEyeContents" ( "id" uuid NOT NULL, "platform" text NOT NULL, "url" text NOT NULL, "post" jsonb NOT NULL, "tenantId" uuid NOT NULL, + "postedAt" timestamptz NOT NULL, "createdAt" timestamptz NOT NULL, "updatedAt" timestamptz NOT NULL, - CONSTRAINT "eageEyeContents_pkey" PRIMARY KEY ("id") + CONSTRAINT "eagleEyeContents_pkey" PRIMARY KEY ("id") ); -ALTER TABLE public."eageEyeContents" ADD CONSTRAINT "eageEyeContents_tenantId_fkey" FOREIGN KEY ("tenantId") REFERENCES public.tenants(id) ON DELETE NO ACTION ON UPDATE NO ACTION; +ALTER TABLE public."eagleEyeContents" ADD CONSTRAINT "eagleEyeContents_tenantId_fkey" FOREIGN KEY ("tenantId") REFERENCES public.tenants(id) ON DELETE NO ACTION ON UPDATE NO ACTION; -CREATE TYPE public."eagleEyeActions_action_type" AS ENUM ('thumbs-up', 'thumbs-down', 'bookmark'); +CREATE TYPE public."eagleEyeActionTypes_type" AS ENUM ('thumbs-up', 'thumbs-down', 'bookmark'); -CREATE TABLE public."eageEyeActions" ( +CREATE TABLE public."eagleEyeActions" ( "id" uuid NOT NULL, - "action" public."eagleEyeActions_action_type" NOT NULL, + "type" public."eagleEyeActionTypes_type" NOT NULL, "timestamp" timestamptz NOT NULL, "contentId" uuid NOT NULL, "tenantId" uuid NOT NULL, "actionById" uuid NOT NULL, "createdAt" timestamptz NOT NULL, "updatedAt" timestamptz NOT NULL, - CONSTRAINT "eageEyeActions_pkey" PRIMARY KEY ("id") + CONSTRAINT "eagleEyeActions_pkey" PRIMARY KEY ("id") ); -ALTER TABLE public."eageEyeActions" ADD CONSTRAINT "eageEyeActions_tenantId_fkey" FOREIGN KEY ("tenantId") REFERENCES public.tenants(id) ON DELETE NO ACTION ON UPDATE NO ACTION; -ALTER TABLE public."eageEyeActions" ADD CONSTRAINT "eageEyeActions_actionBy_fkey" FOREIGN KEY ("actionById") REFERENCES public.users(id) ON DELETE NO ACTION ON UPDATE NO ACTION; -ALTER TABLE public."eageEyeActions" ADD CONSTRAINT "eageEyeActions_contentId_fkey" FOREIGN KEY ("contentId") REFERENCES public."eageEyeContents"(id) ON DELETE NO ACTION ON UPDATE NO ACTION; +ALTER TABLE public."eagleEyeActions" ADD CONSTRAINT "eagleEyeActions_tenantId_fkey" FOREIGN KEY ("tenantId") REFERENCES public.tenants(id) ON DELETE NO ACTION ON UPDATE NO ACTION; +ALTER TABLE public."eagleEyeActions" ADD CONSTRAINT "eagleEyeActions_actionBy_fkey" FOREIGN KEY ("actionById") REFERENCES public.users(id) ON DELETE NO ACTION ON UPDATE NO ACTION; +ALTER TABLE public."eagleEyeActions" ADD CONSTRAINT "eagleEyeActions_contentId_fkey" FOREIGN KEY ("contentId") REFERENCES public."eagleEyeContents"(id) ON DELETE CASCADE ON UPDATE NO ACTION; diff --git a/backend/src/database/models/eagleEyeAction.ts b/backend/src/database/models/eagleEyeAction.ts index b47d8126b0..14eaab4260 100644 --- a/backend/src/database/models/eagleEyeAction.ts +++ b/backend/src/database/models/eagleEyeAction.ts @@ -6,7 +6,7 @@ const eagleEyeActionModel = { defaultValue: DataTypes.UUIDV4, primaryKey: true, }, - action: { + type: { type: DataTypes.TEXT, validate: { isIn: [['thumbs-up', 'thumbs-down', 'bookmark']], @@ -26,7 +26,7 @@ export default (sequelize) => { }) eagleEyeAction.associate = (models) => { - models.eagleEyeContent.belongsTo(models.tenant, { + models.eagleEyeAction.belongsTo(models.tenant, { as: 'tenant', foreignKey: { allowNull: false, diff --git a/backend/src/database/models/eagleEyeContent.ts b/backend/src/database/models/eagleEyeContent.ts index 8b11be5333..198f28e754 100644 --- a/backend/src/database/models/eagleEyeContent.ts +++ b/backend/src/database/models/eagleEyeContent.ts @@ -23,13 +23,17 @@ const eagleEyeContentModel = { validate: { notEmpty: true, }, - } + }, + postedAt: { + type: DataTypes.DATE, + allowNull: false, + }, } export default (sequelize) => { const eagleEyeContent = sequelize.define('eagleEyeContent', eagleEyeContentModel, { timestamps: true, - paranoid: true, + paranoid: false, }) eagleEyeContent.associate = (models) => { @@ -39,6 +43,10 @@ export default (sequelize) => { allowNull: false, }, }) + models.eagleEyeContent.hasMany(models.eagleEyeAction, { + as: 'actions', + foreignKey: 'contentId', + }) } diff --git a/backend/src/database/models/index.ts b/backend/src/database/models/index.ts index 9a1dcc704c..4e0b3bcc09 100644 --- a/backend/src/database/models/index.ts +++ b/backend/src/database/models/index.ts @@ -102,6 +102,7 @@ function models() { require('./conversation').default, require('./conversationSettings').default, require('./eagleEyeContent').default, + require('./eagleEyeAction').default, require('./automation').default, require('./automationExecution').default, require('./organization').default, diff --git a/backend/src/database/repositories/__tests__/eagleEyeActionRepository.test.ts b/backend/src/database/repositories/__tests__/eagleEyeActionRepository.test.ts new file mode 100644 index 0000000000..f3e035cc20 --- /dev/null +++ b/backend/src/database/repositories/__tests__/eagleEyeActionRepository.test.ts @@ -0,0 +1,61 @@ +import EagleEyeContentRepository from '../eagleEyeContentRepository' +import SequelizeTestUtils from '../../utils/sequelizeTestUtils' +import { EagleEyeAction, EagleEyeActionType, EagleEyeContent } from '../../../types/eagleEyeTypes' +import EagleEyeActionRepository from '../eagleEyeActionRepository' + +const db = null + +describe('eagleEyeActionRepository tests', () => { + beforeEach(async () => { + await SequelizeTestUtils.wipeDatabase(db) + }) + + afterAll((done) => { + // Closing the DB connection allows Jest to exit successfully. + SequelizeTestUtils.closeConnection(db) + done() + }) + + describe('createActionForContent method', () => { + it('Should create a an action for a content succesfully', async () => { + const mockIRepositoryOptions = await SequelizeTestUtils.getTestIRepositoryOptions(db) + + const content = { + platform: 'reddit', + url: 'https://some-post-url', + post: { + title: 'post title', + body: 'post body', + }, + postedAt: '2020-05-27T15:13:30Z', + tenantId: mockIRepositoryOptions.currentTenant.id, + } as EagleEyeContent + + const contentCreated = await EagleEyeContentRepository.create(content, mockIRepositoryOptions) + + + const action: EagleEyeAction= { + type: EagleEyeActionType.BOOKMARK, + timestamp: '2022-07-27T19:13:30Z', + } + + const actionCreated = await EagleEyeActionRepository.createActionForContent(action, contentCreated.id, mockIRepositoryOptions) + + actionCreated.createdAt = (actionCreated.createdAt as Date).toISOString().split('T')[0] + actionCreated.updatedAt = (actionCreated.updatedAt as Date).toISOString().split('T')[0] + + const expectedAction = { + id: actionCreated.id, + ...action, + timestamp: new Date(actionCreated.timestamp), + contentId: contentCreated.id, + actionById: mockIRepositoryOptions.currentUser.id, + tenantId: mockIRepositoryOptions.currentTenant.id, + createdAt: SequelizeTestUtils.getNowWithoutTime(), + updatedAt: SequelizeTestUtils.getNowWithoutTime(), + } + expect(expectedAction).toStrictEqual(actionCreated) + }) + + }) +}) diff --git a/backend/src/database/repositories/__tests__/eagleEyeContentRepository.test.ts b/backend/src/database/repositories/__tests__/eagleEyeContentRepository.test.ts index bffec870ca..eef92a3679 100644 --- a/backend/src/database/repositories/__tests__/eagleEyeContentRepository.test.ts +++ b/backend/src/database/repositories/__tests__/eagleEyeContentRepository.test.ts @@ -1,85 +1,11 @@ -import lodash from 'lodash' -import moment from 'moment' import EagleEyeContentRepository from '../eagleEyeContentRepository' import SequelizeTestUtils from '../../utils/sequelizeTestUtils' -import Error404 from '../../../errors/Error404' -import Error400 from '../../../errors/Error400' -import EagleEyeContentService from '../../../services/eagleEyeContentService' -import { PlatformType } from '../../../types/integrationEnums' +import { EagleEyeActionType, EagleEyeContent } from '../../../types/eagleEyeTypes' +import EagleEyeActionRepository from '../eagleEyeActionRepository' + -const db = null -const toCreate = { - sourceId: 'sourceId', - vectorId: '123', - status: null, - platform: 'hacker_news', - title: 'title', - userAttributes: { [PlatformType.GITHUB]: 'hey', [PlatformType.TWITTER]: 'ho' }, - text: 'text', - postAttributes: { - score: 10, - }, - url: 'url', - exactKeywords: null, - timestamp: new Date(), - username: 'username', - keywords: ['keyword1', 'keyword2'], - similarityScore: 0.9, -} - -const toCreateMinimal = { - sourceId: 'sourceIdMinimal', - vectorId: '456', - platform: 'hacker_news', - url: 'url', - title: 'title minimal', - timestamp: new Date(), - username: 'username', - keywords: 'keyword', -} - -const forFiltering = [ - toCreate, - toCreateMinimal, - { - sourceId: 'devto123', - vectorId: '123123', - status: 'engaged', - url: 'devto url', - username: 'devtousername1', - platform: 'devto', - timestamp: moment().toDate(), - title: 'title devto 1', - }, - { - sourceId: 'devto456', - vectorId: '123456', - url: 'url devto 2', - username: 'devtousername2', - status: 'rejected', - platform: 'devto', - timestamp: moment().subtract(1, 'week').toDate(), - title: 'title devto 2', - keywords: ['keyword1', 'keyword2'], - score: 40, - }, - { - sourceId: 'devto789', - vectorId: '123456', - url: 'url devto 3', - username: 'devtousername3', - status: null, - platform: 'devto', - timestamp: moment().subtract(1, 'week').toDate(), - keywords: ['keyword3', 'keyword2'], - title: 'title devto 3', - }, -] - -async function addAll(options) { - await Promise.all(forFiltering.map((item) => EagleEyeContentRepository.upsert(item, options))) -} +const db = null describe('eagleEyeContentRepository tests', () => { beforeEach(async () => { @@ -92,29 +18,40 @@ describe('eagleEyeContentRepository tests', () => { done() }) - describe('upserts method', () => { - it('Should create a complete content succesfully', async () => { + describe('create method', () => { + it('Should create a content succesfully', async () => { const mockIRepositoryOptions = await SequelizeTestUtils.getTestIRepositoryOptions(db) - const created = await EagleEyeContentRepository.upsert(toCreate, mockIRepositoryOptions) + const content = { + platform: 'reddit', + url: 'https://some-post-url', + post: { + title: 'post title', + body: 'post body', + }, + postedAt: '2020-05-27T15:13:30Z', + tenantId: mockIRepositoryOptions.currentTenant.id, + } as EagleEyeContent - created.createdAt = created.createdAt.toISOString().split('T')[0] - created.updatedAt = created.updatedAt.toISOString().split('T')[0] + const created = await EagleEyeContentRepository.create(content, mockIRepositoryOptions) + + created.createdAt = (created.createdAt as Date).toISOString().split('T')[0] + created.updatedAt = (created.updatedAt as Date).toISOString().split('T')[0] const expectedCreated = { id: created.id, - ...toCreate, - importHash: null, + ...content, + postedAt: new Date(content.postedAt), + actions: [], createdAt: SequelizeTestUtils.getNowWithoutTime(), updatedAt: SequelizeTestUtils.getNowWithoutTime(), - deletedAt: null, tenantId: mockIRepositoryOptions.currentTenant.id, - createdById: mockIRepositoryOptions.currentUser.id, - updatedById: mockIRepositoryOptions.currentUser.id, } expect(created).toStrictEqual(expectedCreated) }) + /* + it('Should create a content with unix timestamp', async () => { const mockIRepositoryOptions = await SequelizeTestUtils.getTestIRepositoryOptions(db) @@ -157,130 +94,9 @@ describe('eagleEyeContentRepository tests', () => { expect(created).toStrictEqual(expectedCreated) }) - it('Should not add it the record already exists', async () => { - const mockIRepositoryOptions = await SequelizeTestUtils.getTestIRepositoryOptions(db) - - await EagleEyeContentRepository.upsert(toCreate, mockIRepositoryOptions) - - await EagleEyeContentRepository.upsert(toCreate, mockIRepositoryOptions) - - const count = await mockIRepositoryOptions.database.eagleEyeContent.count() - expect(count).toBe(1) - }) - - it('Should update keywords and similarity score if the item already exists', async () => { - const mockIRepositoryOptions = await SequelizeTestUtils.getTestIRepositoryOptions(db) - - await EagleEyeContentRepository.upsert(toCreate, mockIRepositoryOptions) - - const toCreateNewKeywords = { ...toCreate } - toCreateNewKeywords.keywords = ['1', '2', 'keyword1'] - toCreateNewKeywords.similarityScore = 0.95 - - const allKeywords = ['1', '2', 'keyword1', 'keyword2'] - - const created = await EagleEyeContentRepository.upsert( - toCreateNewKeywords, - mockIRepositoryOptions, - ) - - const count = await mockIRepositoryOptions.database.eagleEyeContent.count() - expect(count).toBe(1) - expect(lodash.isEqual(created.keywords.sort(), allKeywords.sort())) - expect(created.similarityScore).toBe(0.95) - }) - - it('Should create a minimal content succesfully', async () => { - const mockIRepositoryOptions = await SequelizeTestUtils.getTestIRepositoryOptions(db) - - const created = await EagleEyeContentRepository.upsert( - toCreateMinimal, - mockIRepositoryOptions, - ) - - created.createdAt = created.createdAt.toISOString().split('T')[0] - created.updatedAt = created.updatedAt.toISOString().split('T')[0] - - const expectedCreated = { - id: created.id, - ...toCreateMinimal, - text: null, - status: null, - userAttributes: null, - postAttributes: null, - similarityScore: null, - exactKeywords: null, - importHash: null, - createdAt: SequelizeTestUtils.getNowWithoutTime(), - updatedAt: SequelizeTestUtils.getNowWithoutTime(), - deletedAt: null, - tenantId: mockIRepositoryOptions.currentTenant.id, - createdById: mockIRepositoryOptions.currentUser.id, - updatedById: mockIRepositoryOptions.currentUser.id, - } - expect(created).toStrictEqual(expectedCreated) - expect(created.status).toBe(null) - }) - - it('Should create with rejected status', async () => { - const mockIRepositoryOptions = await SequelizeTestUtils.getTestIRepositoryOptions(db) - const newStatus = { ...toCreate } - newStatus.status = 'rejected' - - const created = await EagleEyeContentRepository.upsert(newStatus, mockIRepositoryOptions) - - created.createdAt = created.createdAt.toISOString().split('T')[0] - created.updatedAt = created.updatedAt.toISOString().split('T')[0] - - const expectedCreated = { - id: created.id, - ...newStatus, - importHash: null, - createdAt: SequelizeTestUtils.getNowWithoutTime(), - updatedAt: SequelizeTestUtils.getNowWithoutTime(), - deletedAt: null, - tenantId: mockIRepositoryOptions.currentTenant.id, - createdById: mockIRepositoryOptions.currentUser.id, - updatedById: mockIRepositoryOptions.currentUser.id, - } - expect(created).toStrictEqual(expectedCreated) - expect(created.status).toBe('rejected') - }) - - it('Should create with engaged status', async () => { - const mockIRepositoryOptions = await SequelizeTestUtils.getTestIRepositoryOptions(db) - const newStatus = { ...toCreate } - newStatus.status = 'engaged' - - const created = await EagleEyeContentRepository.upsert(newStatus, mockIRepositoryOptions) - - created.createdAt = created.createdAt.toISOString().split('T')[0] - created.updatedAt = created.updatedAt.toISOString().split('T')[0] - - const expectedCreated = { - id: created.id, - ...newStatus, - importHash: null, - createdAt: SequelizeTestUtils.getNowWithoutTime(), - updatedAt: SequelizeTestUtils.getNowWithoutTime(), - deletedAt: null, - tenantId: mockIRepositoryOptions.currentTenant.id, - createdById: mockIRepositoryOptions.currentUser.id, - updatedById: mockIRepositoryOptions.currentUser.id, - } - expect(created).toStrictEqual(expectedCreated) - expect(created.status).toBe('engaged') - }) - - it('Should throw an error for an invalid status', async () => { - const mockIRepositoryOptions = await SequelizeTestUtils.getTestIRepositoryOptions(db) - const newStatus = { ...toCreate } - newStatus.status = 'smth else' + - await expect(() => - EagleEyeContentRepository.upsert(newStatus, mockIRepositoryOptions), - ).rejects.toThrowError(new Error400('en', 'errors.invalidEagleEyeStatus.message')) - }) + }) describe('find by id method', () => { @@ -601,4 +417,137 @@ describe('eagleEyeContentRepository tests', () => { expect(updated.keywords).toStrictEqual(created.keywords) }) }) + */ + }) + + describe('findAndCountAll method', () => { + it('Should find eagle eye contant, various cases', async () => { + // create random tenant with one user + const mockIRepositoryOptions = await SequelizeTestUtils.getTestIRepositoryOptions(db) + + // create additional users for same tenant to test out actionBy filtering + const randomUser = await SequelizeTestUtils.getRandomUser() + + console.log("random user: ") + console.log(randomUser) + + const user2 = await mockIRepositoryOptions.database.user.create(randomUser) + + await mockIRepositoryOptions.database.tenantUser.create({ + roles: ['admin'], + status: 'active', + tenantId: mockIRepositoryOptions.currentTenant.id, + userId: user2.id, + }) + + // create few content + // one without any actions + await EagleEyeContentRepository.create( + { + platform: 'reddit', + url: 'https://some-reddit-url', + post: { + title: 'post title', + body: 'post body', + }, + postedAt: '2020-05-27T15:13:30Z', + tenantId: mockIRepositoryOptions.currentTenant.id, + }, + mockIRepositoryOptions, + ) + + + // one with a bookmark action + let c2 = await EagleEyeContentRepository.create( + { + platform: 'hackernews', + url: 'https://some-hackernews-url', + post: { + title: 'post title', + body: 'post body', + }, + postedAt: '2022-06-27T19:14:44Z', + tenantId: mockIRepositoryOptions.currentTenant.id, + }, + mockIRepositoryOptions, + ) + + // add bookmark action + await EagleEyeActionRepository.createActionForContent( + { + type: EagleEyeActionType.BOOKMARK, + timestamp: '2022-07-27T19:13:30Z', + }, + c2.id, + mockIRepositoryOptions, + ) + + c2 = await EagleEyeContentRepository.findById(c2.id, mockIRepositoryOptions) + + // another content with a thumbs-up(user1) and a bookmark(user2) action + let c3 = await EagleEyeContentRepository.create( + { + platform: 'devto', + url: 'https://some-devto-url', + post: { + title: 'post title', + body: 'post body', + }, + postedAt: '2022-06-27T19:14:44Z', + tenantId: mockIRepositoryOptions.currentTenant.id, + }, + mockIRepositoryOptions, + ) + + // add the thumbs up action + await EagleEyeActionRepository.createActionForContent( + { + type: EagleEyeActionType.THUMBS_UP, + timestamp: '2022-09-30T23:11:10Z', + }, + c3.id, + mockIRepositoryOptions, + ) + + // also add bookmark from user2 + await EagleEyeActionRepository.createActionForContent( + { + type: EagleEyeActionType.BOOKMARK, + timestamp: '2022-09-30T23:11:10Z', + }, + c3.id, + {...mockIRepositoryOptions, currentUser: user2}, + ) + + c3 = await EagleEyeContentRepository.findById(c3.id, mockIRepositoryOptions) + + + // filter by action type + let res = await EagleEyeContentRepository.findAndCountAll({ + advancedFilter: { + action: { + type: EagleEyeActionType.BOOKMARK + + } + } + }, mockIRepositoryOptions) + + + expect(res.count).toBe(2) + expect(res.rows.sort((a, b) => (a.createdAt > b.createdAt ? 1 : -1))).toStrictEqual([c2,c3]) + + // filter by actionBy + res = await EagleEyeContentRepository.findAndCountAll({ + advancedFilter: { + action: { + actionById: user2.id + } + } + }, mockIRepositoryOptions) + + expect(res.count).toBe(1) + expect(res.rows).toStrictEqual([c3]) + + }) + }) }) diff --git a/backend/src/database/repositories/eagleEyeActionRepository.ts b/backend/src/database/repositories/eagleEyeActionRepository.ts index 8fb48cce8b..cc7932b257 100644 --- a/backend/src/database/repositories/eagleEyeActionRepository.ts +++ b/backend/src/database/repositories/eagleEyeActionRepository.ts @@ -1,7 +1,125 @@ +import lodash from 'lodash' +import Error404 from '../../errors/Error404' +import { EagleEyeAction, EagleEyeActionType } from '../../types/eagleEyeTypes' import { IRepositoryOptions } from './IRepositoryOptions' +import SequelizeRepository from './sequelizeRepository' export default class EagleEyeActionRepository { + static async createActionForContent( + data: EagleEyeAction, + contentId: string, + options: IRepositoryOptions, + ): Promise { + const currentUser = SequelizeRepository.getCurrentUser(options) - + const transaction = SequelizeRepository.getTransaction(options) -} \ No newline at end of file + const currentTenant = SequelizeRepository.getCurrentTenant(options) + + const record = await options.database.eagleEyeAction.create( + { + ...lodash.pick(data, ['type', 'timestamp']), + actionById: currentUser.id, + contentId, + tenantId: currentTenant.id, + }, + { + transaction, + }, + ) + + return this.findById(record.id, options) + } + + static async removeActionFromContent( + action: EagleEyeActionType, + contentId: string, + options: IRepositoryOptions, + ) { + const currentUser = SequelizeRepository.getCurrentUser(options) + + const currentTenant = SequelizeRepository.getCurrentTenant(options) + + const transaction = SequelizeRepository.getTransaction(options) + + const record = await options.database.eagleEyeAction.findOne({ + where: { + contentId, + action, + actionById: currentUser.id, + tenantId: currentTenant.id, + }, + transaction, + }) + + if (record) { + await record.destroy({ + transaction, + force: true, + }) + } + } + + static async destroy(id: string, options: IRepositoryOptions): Promise { + const transaction = SequelizeRepository.getTransaction(options) + + const currentTenant = SequelizeRepository.getCurrentTenant(options) + + const record = await options.database.eagleEyeAction.findOne({ + where: { + id, + tenantId: currentTenant.id, + }, + transaction, + }) + + if (record) { + await record.destroy({ + transaction, + force: true, + }) + } + } + + static async findById(id: string, options: IRepositoryOptions) { + const transaction = SequelizeRepository.getTransaction(options) + + const include = [] + + const currentTenant = SequelizeRepository.getCurrentTenant(options) + + const record = await options.database.eagleEyeAction.findOne({ + where: { + id, + tenantId: currentTenant.id, + }, + include, + transaction, + }) + + if (!record) { + throw new Error404() + } + + return this._populateRelations(record) + } + + static async create(data: EagleEyeAction, options: IRepositoryOptions): Promise { + const currentTenant = SequelizeRepository.getCurrentTenant(options) + + const record = options.database.eagleEyeContent.create({ + ...lodash.pick(data, ['type', 'timestamp']), + tenantId: currentTenant.id, + }) + + return this.findById(record.id, options) + } + + static async _populateRelations(record) { + if (!record) { + return record + } + + return record.get({ plain: true }) + } +} diff --git a/backend/src/database/repositories/eagleEyeContentRepository.ts b/backend/src/database/repositories/eagleEyeContentRepository.ts index ca7aff5eef..73dcf99ee5 100644 --- a/backend/src/database/repositories/eagleEyeContentRepository.ts +++ b/backend/src/database/repositories/eagleEyeContentRepository.ts @@ -1,60 +1,39 @@ -import moment from 'moment' import lodash from 'lodash' +import { Op } from 'sequelize' import SequelizeRepository from './sequelizeRepository' import Error404 from '../../errors/Error404' -import Error400 from '../../errors/Error400' -import AuditLogRepository from './auditLogRepository' import { IRepositoryOptions } from './IRepositoryOptions' -import { EagleEyeContentData } from '../../types/eagleEyeTypes' +import { EagleEyeContent } from '../../types/eagleEyeTypes' +import QueryParser from './filters/queryParser' +import { QueryOutput } from './filters/queryTypes' +import SequelizeFilterUtils from '../utils/sequelizeFilterUtils' +import EagleEyeActionRepository from './eagleEyeActionRepository' export default class EagleEyeContentRepository { - /** - * Create an eagle eye shown content record. - * @param data Data to a new EagleEyeContent record. - * @param options Repository options. - * @returns Created EagleEyeContent record. - */ - static async upsert(data:EagleEyeContentData, options: IRepositoryOptions): Promise { - - if(!data.url){ - throw new Error(`Can't upsert without url`) - } - // find by url - const existing = await EagleEyeContentRepository.findByUrl(data.url, options) + static async create(data:EagleEyeContent, options:IRepositoryOptions): Promise{ + const currentTenant = SequelizeRepository.getCurrentTenant(options) - let record + const record = await options.database.eagleEyeContent.create( + { + ...lodash.pick(data, [ + 'platform', + 'post', + 'url', + 'postedAt' + ]), + tenantId: currentTenant.id, + }, + ) - if (existing){ - record = await EagleEyeContentRepository.update(existing.id, data, options) - } - /* - else{ - record = options.database.eagleEyeContent.create( - { - ...lodash.pick(data, [ - 'platform', - 'post', - 'url', - ]), - memberId: data.member || null, - parentId: data.parent || null, - sourceParentId: data.sourceParentId || null, - conversationId: data.conversationId || null, - tenantId: tenant.id, - createdById: currentUser.id, - updatedById: currentUser.id, - }, - { - transaction, - }, - ) + if (data.actions){ + for (const action of data.actions){ + await EagleEyeActionRepository.createActionForContent(action, record.id, options) + } } - */ - - return this.findById(record.id, options) + } @@ -83,6 +62,7 @@ export default class EagleEyeContentRepository { ...lodash.pick(data, [ 'platform', 'post', + 'postedAt', 'url', ]), updatedById: currentUser.id, @@ -98,7 +78,14 @@ export default class EagleEyeContentRepository { static async findById(id:string, options: IRepositoryOptions) { const transaction = SequelizeRepository.getTransaction(options) - const include = [] + const include = [{ + model: options.database.eagleEyeAction, + as: 'actions', + // attributes: [], + // through: { + // attributes: [], + // }, + }] const currentTenant = SequelizeRepository.getCurrentTenant(options) @@ -118,6 +105,131 @@ export default class EagleEyeContentRepository { return this._populateRelations(record) } + static async destroy(id: string, options: IRepositoryOptions): Promise { + const transaction = SequelizeRepository.getTransaction(options) + + const currentTenant = SequelizeRepository.getCurrentTenant(options) + + const record = await options.database.eagleEyeContent.findOne({ + where: { + id, + tenantId: currentTenant.id, + }, + transaction, + }) + + if (record) { + await record.destroy({ + transaction, + force: true, + }) + } + } + + static async findAndCountAll( + { + advancedFilter = null as any, + limit = 0, + offset = 0, + orderBy = '', + }, + options: IRepositoryOptions, + ) { + + const actionsSequelizeInclude = { + model: options.database.eagleEyeAction, + as: 'actions', + // required:false, + where: {}, + // subQuery:true + + } + + // let wh = {} + + if (advancedFilter && advancedFilter.action) { + + const actionQueryParser = new QueryParser( + { + // nestedFields: { + // type: `$actions.type$` + // } + }, + options, + ) + + const parsedActionQuery: QueryOutput = actionQueryParser.parse({ + filter: advancedFilter.action, + orderBy: 'timestamp_DESC', + }) + + actionsSequelizeInclude.where = parsedActionQuery.where ?? {} + delete advancedFilter.action + } + + const include = [ + actionsSequelizeInclude, + ] + + const contentParser = new QueryParser( + {}, + options, + ) + + + const parsed: QueryOutput = contentParser.parse({ + filter: advancedFilter, + orderBy: orderBy || ['postedAt_DESC'], + limit, + offset, + }) + + + + console.log("SENDING SHIEEEEEEEEEET") + console.log(parsed) + console.log(include) + + // const wtf = parsed.where ? { where: {...parsed.where, ...wh} } : {} + console.log("wtf") + // console.log(wtf) + let { + rows, + count, // eslint-disable-line prefer-const + } = await options.database.eagleEyeContent.findAndCountAll({ + include, + ...(parsed.where ? { where: parsed.where } : {}), + order: parsed.order, + limit: parsed.limit, + offset: parsed.offset, + transaction: SequelizeRepository.getTransaction(options), + subQuery: false + }) + + // If we have an actions filter, we should query again to eager + // load the all actions on a content because previous query will + // omit actions that don't match the given action filter + if (Object.keys(actionsSequelizeInclude.where).length !== 0) + { + rows = (await options.database.eagleEyeContent.findAndCountAll({ + include: [{...actionsSequelizeInclude, where: {}}], + where: { id: { [Op.in]: rows.map( (i) => i.id) } }, + order: parsed.order, + limit: parsed.limit, + offset: parsed.offset, + transaction: SequelizeRepository.getTransaction(options), + subQuery: false + + })).rows + } + + + rows = await this._populateRelationsForRows(rows) + + return { rows, count, limit: parsed.limit, offset: parsed.offset } + + } + static async findByUrl(url:string, options: IRepositoryOptions) { const transaction = SequelizeRepository.getTransaction(options) diff --git a/backend/src/database/utils/sequelizeTestUtils.ts b/backend/src/database/utils/sequelizeTestUtils.ts index 7b103981f4..83e813d9a2 100644 --- a/backend/src/database/utils/sequelizeTestUtils.ts +++ b/backend/src/database/utils/sequelizeTestUtils.ts @@ -34,6 +34,7 @@ export default class SequelizeTestUtils { files, microservices, "eagleEyeContents", + "eagleEyeActions", "auditLogs", "memberEnrichmentCache" cascade; diff --git a/backend/src/services/__tests__/eagleEyeContentService.test.ts b/backend/src/services/__tests__/eagleEyeContentService.test.ts index 46b7d51a8b..f67db5118e 100644 --- a/backend/src/services/__tests__/eagleEyeContentService.test.ts +++ b/backend/src/services/__tests__/eagleEyeContentService.test.ts @@ -1,44 +1,9 @@ -import moment from 'moment' import SequelizeTestUtils from '../../database/utils/sequelizeTestUtils' -import { PlatformType } from '../../types/integrationEnums' +import { EagleEyeActionType, EagleEyeContent } from '../../types/eagleEyeTypes' import EagleEyeContentService from '../eagleEyeContentService' const db = null -const toUpsert = { - keywords: ['keyword'], - similarityScore: 1, - userAttributes: { - [PlatformType.GITHUB]: 'github', - }, - username: 'username', - timestamp: moment().unix(), - url: 'url', - title: 'title', - text: 'text', - platform: 'platform', - sourceId: 'sourceId', - vectorId: '1234', - status: null, -} - -const toUpsert2 = { - keywords: ['keyword'], - similarityScore: 1, - userAttributes: { - [PlatformType.GITHUB]: 'github', - }, - username: 'username', - timestamp: moment().subtract(1, 'days').unix(), - url: 'url', - title: 'title', - text: 'text', - platform: 'platform', - sourceId: 'sourceId2', - vectorId: '12345', - status: null, -} - describe('EagleEyeContentService tests', () => { beforeEach(async () => { await SequelizeTestUtils.wipeDatabase(db) @@ -49,91 +14,57 @@ describe('EagleEyeContentService tests', () => { await SequelizeTestUtils.closeConnection(db) }) - describe('bulk upsert method', () => { - it('Should upsert a single record', async () => { + describe('upsert method', () => { + it('Should create or update a single content using URL field', async () => { const mockIRepositoryOptions = await SequelizeTestUtils.getTestIRepositoryOptions(db) - const service = new EagleEyeContentService(mockIRepositoryOptions) - await service.bulkUpsert([toUpsert]) - const result = await service.findAndCountAll({}) - expect(result.count).toBe(1) - }) - it('Should upsert a single record', async () => { - const mockIRepositoryOptions = await SequelizeTestUtils.getTestIRepositoryOptions(db) - const service = new EagleEyeContentService(mockIRepositoryOptions) - await service.bulkUpsert([toUpsert]) - await service.bulkUpsert([toUpsert2]) - const result = await service.findAndCountAll({}) - expect(result.count).toBe(2) - }) - }) - - describe('findAndCount all method', () => { - it('Should find records', async () => { - const mockIRepositoryOptions = await SequelizeTestUtils.getTestIRepositoryOptions(db) - const service = new EagleEyeContentService(mockIRepositoryOptions) - await service.bulkUpsert([toUpsert]) - await service.bulkUpsert([toUpsert2]) - const result = await service.findAndCountAll({}) - - expect(result.count).toBe(2) - expect(result.rows[1].vectorId).toBe(toUpsert.vectorId) - expect(result.rows[0].vectorId).toBe(toUpsert2.vectorId) - }) + const content: EagleEyeContent = { + platform: 'reddit', + url: 'https://some-post-url', + post: { + title: 'post title', + body: 'post body', + }, + postedAt: '2020-05-27T15:13:30Z', + tenantId: mockIRepositoryOptions.currentTenant.id, + actions: [{ + type: EagleEyeActionType.BOOKMARK, + timestamp: '2022-06-27T14:13:30Z', + }] + } - it('Should work when no records', async () => { - const mockIRepositoryOptions = await SequelizeTestUtils.getTestIRepositoryOptions(db) const service = new EagleEyeContentService(mockIRepositoryOptions) - const result = await service.findAndCountAll({}) - expect(result.count).toBe(0) - }) - }) + const c1 = await service.upsert(content) - describe('findNotInbox method', () => { - it('4 records: 2 have status null, one is too old. Return 1', async () => { - const mockIRepositoryOptions = await SequelizeTestUtils.getTestIRepositoryOptions(db) - const service = new EagleEyeContentService(mockIRepositoryOptions) + let contents = await service.query({}) - const nInbox1 = { - keywords: ['keyword'], - similarityScore: 1, - userAttributes: { - [PlatformType.GITHUB]: 'github', - }, - username: 'username', - timestamp: moment().unix(), - url: 'url', - title: 'title', - text: 'text', - platform: 'platform', - sourceId: 'p-321', - vectorId: '321', - status: 'rejected', - } + expect(contents.count).toBe(1) + expect(contents.rows).toStrictEqual([c1]) - const nInbox2 = { - keywords: ['keyword'], - similarityScore: 1, - userAttributes: { - [PlatformType.GITHUB]: 'github', + // upsert previous url with some new fields + const contentWithSameUrl: EagleEyeContent = { + platform: 'reddit', + url: 'https://some-post-url', + post: { + title: 'a brand new post title', + body: 'better post body', }, - username: 'username', - timestamp: moment().subtract(31, 'days').unix(), - url: 'url', - title: 'title', - text: 'text', - platform: 'platform', - sourceId: 'p-4321', - vectorId: '4321', - status: 'engaged', + postedAt: '2020-05-27T15:13:30Z', + tenantId: mockIRepositoryOptions.currentTenant.id, } - await service.bulkUpsert([toUpsert, nInbox1, nInbox2, toUpsert2]) + const c1Upserted = await service.upsert(contentWithSameUrl) - const result = await service.findNotInbox() + contents = await service.query({}) + expect(contents.count).toBe(1) + expect(contents.rows).toStrictEqual([c1Upserted]) + expect(c1Upserted.id).toEqual(c1.id) + expect(contents.rows[0].post).toStrictEqual(contentWithSameUrl.post) - expect(result.length).toBe(1) - expect(result[0]).toBe(nInbox1.vectorId) }) + + }) + + }) diff --git a/backend/src/services/eagleEyeActionService.ts b/backend/src/services/eagleEyeActionService.ts new file mode 100644 index 0000000000..7711915543 --- /dev/null +++ b/backend/src/services/eagleEyeActionService.ts @@ -0,0 +1,79 @@ +import EagleEyeActionRepository from '../database/repositories/eagleEyeActionRepository' +import EagleEyeContentRepository from '../database/repositories/eagleEyeContentRepository' +import Error404 from '../errors/Error404' +import { EagleEyeAction, EagleEyeActionType } from '../types/eagleEyeTypes' +import { IServiceOptions } from './IServiceOptions' +import { LoggingBase } from './loggingBase' + +export default class EagleEyeActionService extends LoggingBase { + options: IServiceOptions + + constructor(options) { + super(options) + this.options = options + } + + async create(data: EagleEyeAction, contentId: string): Promise { + // find content + const content = await EagleEyeContentRepository.findById(contentId, this.options) + + if (!content) { + throw new Error404('Content not found..') + } + + const existingUserActions: EagleEyeAction[] = content.actions + .filter((a) => a.actionById === this.options.currentUser.id) + + const existingUserActionTypes = existingUserActions.map((a) => a.type) + + // check if already bookmarked - if yes ignore the new action and return existing + if ( + data.type === EagleEyeActionType.BOOKMARK && + existingUserActionTypes.includes(EagleEyeActionType.BOOKMARK) + ) { + return existingUserActions.find( (a) => a.type === EagleEyeActionType.BOOKMARK) + } + + // thumbs up and down should be mutually exclusive + if ( + data.type === EagleEyeActionType.THUMBS_DOWN && + existingUserActionTypes.includes(EagleEyeActionType.THUMBS_UP) + ) { + await EagleEyeActionRepository.removeActionFromContent( + EagleEyeActionType.THUMBS_UP, + contentId, + this.options, + ) + } else if ( + data.type === EagleEyeActionType.THUMBS_UP && + existingUserActionTypes.includes(EagleEyeActionType.THUMBS_DOWN) + ) { + await EagleEyeActionRepository.removeActionFromContent( + EagleEyeActionType.THUMBS_DOWN, + contentId, + this.options, + ) + } + + // add new action + return EagleEyeActionRepository.createActionForContent(data, contentId, this.options) + } + + + async destroy(id: string){ + + const action = await EagleEyeActionRepository.findById(id, this.options) + + const contentId = action.contentId + + await EagleEyeActionRepository.destroy(id, this.options) + + // find content + const content = await EagleEyeContentRepository.findById(contentId, this.options) + + // if content no longer has any actions attached to it, also delete the content + if (content.actions.length === 0){ + await EagleEyeContentRepository.destroy(contentId, this.options) + } + } +} diff --git a/backend/src/services/eagleEyeContentService.ts b/backend/src/services/eagleEyeContentService.ts index e42de0de9a..69bfd7f132 100644 --- a/backend/src/services/eagleEyeContentService.ts +++ b/backend/src/services/eagleEyeContentService.ts @@ -1,30 +1,13 @@ -import moment from 'moment' -import request from 'superagent' -import { API_CONFIG } from '../config' -import SequelizeRepository from '../database/repositories/sequelizeRepository' import { IServiceOptions } from './IServiceOptions' import EagleEyeContentRepository from '../database/repositories/eagleEyeContentRepository' -import Error400 from '../errors/Error400' -import track from '../segment/track' import { LoggingBase } from './loggingBase' +import { EagleEyeContent, EagleEyeAction } from '../types/eagleEyeTypes' +import { PageData, QueryData } from '../types/common' -interface EagleEyeSearchPoint { - vectorId: string - sourceId: string - title: string - text?: string - url: string - timestamp: number - username: string - similarityScore: number - userAttributes: { - [platform: string]: string - } - keywords: string[] +export interface EagleEyeContentUpsertData extends EagleEyeAction { + content: EagleEyeContent } -type EagleEyeSearchOutput = EagleEyeSearchPoint[] - export default class EagleEyeContentService extends LoggingBase { options: IServiceOptions @@ -33,51 +16,36 @@ export default class EagleEyeContentService extends LoggingBase { this.options = options } - async upsert(data) { - const transaction = await SequelizeRepository.createTransaction(this.options) - - try { - const record = await EagleEyeContentRepository.upsert(data, { - ...this.options, - transaction, - }) - - await SequelizeRepository.commitTransaction(transaction) + /** + * Create an eagle eye shown content record. + * @param data Data to a new EagleEyeContent record. + * @param options Repository options. + * @returns Created EagleEyeContent record. + */ + async upsert(data: EagleEyeContent): Promise { + if (!data.url) { + throw new Error(`Can't upsert without url`) + } - return record - } catch (error) { - await SequelizeRepository.rollbackTransaction(transaction) + // find by url + const existing = await EagleEyeContentRepository.findByUrl(data.url, this.options) - SequelizeRepository.handleUniqueFieldError(error, this.options.language, 'EagleEyeContent') + let record - throw error + if (existing) { + record = await EagleEyeContentRepository.update(existing.id, data, this.options) + } else { + record = await EagleEyeContentRepository.create(data, this.options) } - } - async findNotInbox() { - const shown = ( - await EagleEyeContentRepository.findAndCountAll( - { - filter: { - timestampRange: [ - moment().subtract(30, 'days').toDate(), - moment().add(1, 'hour').toDate(), - ], - status: 'NOT_NULL', - }, - }, - this.options, - ) - ).rows - // Slicing results such that lambda payload will not be too big - return shown.map((record) => record.vectorId).slice(0, 20000) + return record } - async findAndCountAll(args) { - return EagleEyeContentRepository.findAndCountAll(args, this.options) + async findById(id: string): Promise { + return EagleEyeContentRepository.findById(id, this.options) } - async query(data) { + async query(data: QueryData): Promise> { const advancedFilter = data.filter const orderBy = data.orderBy const limit = data.limit @@ -88,99 +56,9 @@ export default class EagleEyeContentService extends LoggingBase { ) } - async bulkUpsert(data: EagleEyeSearchOutput) { - for (const point of data) { - await this.upsert(point) - } - } - + /** TODO async search(args) { - const { keywords, nDays, exactKeywords } = args - // We do not want what we have already accepted or rejected - const filters = await this.findNotInbox() - if (API_CONFIG.premiumApiUrl) { - const response = await request - .post(`${API_CONFIG.premiumApiUrl}/search`) - .send({ queries: keywords, nDays, filters, exactKeywords }) - const fromEagleEye: EagleEyeSearchOutput = JSON.parse(response.text) - await this.bulkUpsert(fromEagleEye) - return fromEagleEye - } - return [] as EagleEyeSearchOutput - } - - async keywordMatch(args) { - const { keywords, nDays, platform } = args - - if (API_CONFIG.premiumApiUrl) { - const response = await request - .post(`${API_CONFIG.premiumApiUrl}/keyword-match`) - .send({ exactKeywords: keywords, nDays, platform }) - try { - return JSON.parse(response.text) - } catch (error) { - this.log.error({ error: response.error }, 'error while calling eagle eye server!') - throw new Error400('en', 'errors.eagleEyeSearchFailed.message') - } - } else { - return [] as EagleEyeSearchOutput - } - } - - async update(id, data) { - const transaction = await SequelizeRepository.createTransaction(this.options) - - try { - const recordBeforeUpdate = await EagleEyeContentRepository.findById(id, { ...this.options }) - const record = await EagleEyeContentRepository.update(id, data, { - ...this.options, - transaction, - }) - - // If we are updating status we want to track it - if (data.status !== recordBeforeUpdate.status) { - // If we are going from null to status, we are either accepting or rejecting - if (data.status && data.status !== null && data.status !== undefined) { - track( - `EagleEye ${data.status}`, - { - ...data, - platform: record.platform, - keywords: record.keywords, - title: record.title, - url: record.url, - }, - { ...this.options }, - ) - // Here we are bringing back a rejected post to the Inbox - } else if (recordBeforeUpdate.status === 'rejected' && data.status === null) { - track( - `EagleEye post from rejected to Inbox`, - { - ...data, - platform: record.platform, - keywords: record.keywords, - title: record.title, - url: record.url, - }, - { ...this.options }, - ) - } - } - - await SequelizeRepository.commitTransaction(transaction) - - return record - } catch (error) { - await SequelizeRepository.rollbackTransaction(transaction) - - SequelizeRepository.handleUniqueFieldError(error, this.options.language, 'EagleEyeContent') - - throw error - } - } - - async findById(id) { - return EagleEyeContentRepository.findById(id, this.options) + } + */ } diff --git a/backend/src/types/common.ts b/backend/src/types/common.ts index 1eb58e2dc9..d7b4b7a253 100644 --- a/backend/src/types/common.ts +++ b/backend/src/types/common.ts @@ -5,6 +5,13 @@ export interface PageData { offset: number } +export interface QueryData { + filter?: any + orderBy?: string + limit?: number + offset?: number +} + export interface SearchCriteria { limit?: number offset?: number diff --git a/backend/src/types/eagleEyeTypes.ts b/backend/src/types/eagleEyeTypes.ts index 0ffbaaf1f2..556ed1fd8c 100644 --- a/backend/src/types/eagleEyeTypes.ts +++ b/backend/src/types/eagleEyeTypes.ts @@ -4,21 +4,24 @@ export enum EagleEyeActionType { BOOKMARK = 'bookmark', } -export interface EagleEyeContentData { + +export interface EagleEyeAction { + id?: string + type: EagleEyeActionType + timestamp: Date | string + createdAt?: Date | string + updatedAt?: Date | string +} + +export interface EagleEyeContent { id?: string platform: string post: any url: string + actions?: EagleEyeAction[] tenantId: string - createdAt?: string - updatedAt?: string + postedAt: string + createdAt?: Date | string + updatedAt?: Date | string } -export interface EagleEyeActionData { - id?: string - action: EagleEyeActionType - timestamp: string - content: EagleEyeContentData - createdAt?: string - updatedAt?: string -} From 718311beff185389a452625a3848638316e226c6 Mon Sep 17 00:00:00 2001 From: anilb Date: Sun, 5 Feb 2023 20:00:31 +0100 Subject: [PATCH 06/36] comments --- .../repositories/eagleEyeContentRepository.ts | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/backend/src/database/repositories/eagleEyeContentRepository.ts b/backend/src/database/repositories/eagleEyeContentRepository.ts index 73dcf99ee5..66b87609f6 100644 --- a/backend/src/database/repositories/eagleEyeContentRepository.ts +++ b/backend/src/database/repositories/eagleEyeContentRepository.ts @@ -6,7 +6,6 @@ import { IRepositoryOptions } from './IRepositoryOptions' import { EagleEyeContent } from '../../types/eagleEyeTypes' import QueryParser from './filters/queryParser' import { QueryOutput } from './filters/queryTypes' -import SequelizeFilterUtils from '../utils/sequelizeFilterUtils' import EagleEyeActionRepository from './eagleEyeActionRepository' export default class EagleEyeContentRepository { @@ -81,10 +80,6 @@ export default class EagleEyeContentRepository { const include = [{ model: options.database.eagleEyeAction, as: 'actions', - // attributes: [], - // through: { - // attributes: [], - // }, }] const currentTenant = SequelizeRepository.getCurrentTenant(options) @@ -139,21 +134,15 @@ export default class EagleEyeContentRepository { const actionsSequelizeInclude = { model: options.database.eagleEyeAction, as: 'actions', - // required:false, where: {}, - // subQuery:true } - // let wh = {} if (advancedFilter && advancedFilter.action) { const actionQueryParser = new QueryParser( { - // nestedFields: { - // type: `$actions.type$` - // } }, options, ) @@ -185,14 +174,6 @@ export default class EagleEyeContentRepository { }) - - console.log("SENDING SHIEEEEEEEEEET") - console.log(parsed) - console.log(include) - - // const wtf = parsed.where ? { where: {...parsed.where, ...wh} } : {} - console.log("wtf") - // console.log(wtf) let { rows, count, // eslint-disable-line prefer-const From 7cf414b9bea01e3651c9d86159ba35e27e26d348 Mon Sep 17 00:00:00 2001 From: anilb Date: Sun, 5 Feb 2023 20:33:37 +0100 Subject: [PATCH 07/36] eagleEye upsert routing --- .../src/api/eagleEyeContent/eagleEyeContentUpsert.ts | 11 +++++++++++ backend/src/security/permissions.ts | 5 +++++ 2 files changed, 16 insertions(+) diff --git a/backend/src/api/eagleEyeContent/eagleEyeContentUpsert.ts b/backend/src/api/eagleEyeContent/eagleEyeContentUpsert.ts index e69de29bb2..2acfcc5e2d 100644 --- a/backend/src/api/eagleEyeContent/eagleEyeContentUpsert.ts +++ b/backend/src/api/eagleEyeContent/eagleEyeContentUpsert.ts @@ -0,0 +1,11 @@ +import Permissions from '../../security/permissions' +import EagleEyeContentService from '../../services/eagleEyeContentService' +import PermissionChecker from '../../services/user/permissionChecker' + +export default async (req, res) => { + new PermissionChecker(req).validateHas(Permissions.values.eagleEyeContentCreate) + + const payload = await new EagleEyeContentService(req).upsert(req.body) + + await req.responseHandler.success(req, res, payload) +} diff --git a/backend/src/security/permissions.ts b/backend/src/security/permissions.ts index 3c75df4c2b..74482a3f03 100644 --- a/backend/src/security/permissions.ts +++ b/backend/src/security/permissions.ts @@ -422,6 +422,11 @@ class Permissions { allowedRoles: [roles.admin, roles.readonly], allowedPlans: [plans.essential, plans.growth], }, + eagleEyeContentCreate: { + id: 'eagleEyeContentCreate', + allowedRoles: [roles.admin], + allowedPlans: [plans.growth], + }, eagleEyeContentRead: { id: 'eagleEyeContentRead', allowedRoles: [roles.admin, roles.readonly], From 13079a173df2ff23e0a44dd6cee3500e1c5145fc Mon Sep 17 00:00:00 2001 From: anilb Date: Sun, 5 Feb 2023 20:37:09 +0100 Subject: [PATCH 08/36] transactions for upsert --- .../src/services/eagleEyeContentService.ts | 38 +++++++++++++------ 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/backend/src/services/eagleEyeContentService.ts b/backend/src/services/eagleEyeContentService.ts index 69bfd7f132..e78e2eb93a 100644 --- a/backend/src/services/eagleEyeContentService.ts +++ b/backend/src/services/eagleEyeContentService.ts @@ -3,6 +3,7 @@ import EagleEyeContentRepository from '../database/repositories/eagleEyeContentR import { LoggingBase } from './loggingBase' import { EagleEyeContent, EagleEyeAction } from '../types/eagleEyeTypes' import { PageData, QueryData } from '../types/common' +import SequelizeRepository from '../database/repositories/sequelizeRepository' export interface EagleEyeContentUpsertData extends EagleEyeAction { content: EagleEyeContent @@ -23,22 +24,37 @@ export default class EagleEyeContentService extends LoggingBase { * @returns Created EagleEyeContent record. */ async upsert(data: EagleEyeContent): Promise { - if (!data.url) { - throw new Error(`Can't upsert without url`) - } - // find by url - const existing = await EagleEyeContentRepository.findByUrl(data.url, this.options) + const transaction = await SequelizeRepository.createTransaction(this.options) + + try { + + if (!data.url) { + throw new Error(`Can't upsert without url`) + } + + // find by url + const existing = await EagleEyeContentRepository.findByUrl(data.url, this.options) + + let record + + if (existing) { + record = await EagleEyeContentRepository.update(existing.id, data, this.options) + } else { + record = await EagleEyeContentRepository.create(data, this.options) + } + + await SequelizeRepository.commitTransaction(transaction) + + return record + } catch (error) { + await SequelizeRepository.rollbackTransaction(transaction) - let record + SequelizeRepository.handleUniqueFieldError(error, this.options.language, 'EagleEyeContent') - if (existing) { - record = await EagleEyeContentRepository.update(existing.id, data, this.options) - } else { - record = await EagleEyeContentRepository.create(data, this.options) + throw error } - return record } async findById(id: string): Promise { From ee65fc2433eb396b533cede96cae4bc2b9bd746f Mon Sep 17 00:00:00 2001 From: anilb Date: Sun, 5 Feb 2023 20:46:15 +0100 Subject: [PATCH 09/36] error translations --- backend/src/i18n/en.ts | 10 ++--- .../src/services/eagleEyeContentService.ts | 39 +++++++------------ 2 files changed, 16 insertions(+), 33 deletions(-) diff --git a/backend/src/i18n/en.ts b/backend/src/i18n/en.ts index 0a41b33134..0c4a89b705 100644 --- a/backend/src/i18n/en.ts +++ b/backend/src/i18n/en.ts @@ -102,19 +102,15 @@ const en = { activityDup: { message: 'This activity has already been linked to this member', }, - invalidEagleEyeStatus: { - message: 'Possible statuses are: "shown", "rejected", "engaged"', - }, - eagleEyeSearchFailed: { - message: 'Search failed in EagleEye', - }, OrganizationNameRequired: { message: 'Organization Name is required', }, projectNotFound: { message: 'Project not found', }, - + eagleEye: { + urlRequiredWhenUpserting: 'URL field is mandatory when upserting eagleEyeContent' + }, integrations: { badEndpoint: 'Bad endpoint: {0}', }, diff --git a/backend/src/services/eagleEyeContentService.ts b/backend/src/services/eagleEyeContentService.ts index e78e2eb93a..fdbd133d55 100644 --- a/backend/src/services/eagleEyeContentService.ts +++ b/backend/src/services/eagleEyeContentService.ts @@ -3,7 +3,7 @@ import EagleEyeContentRepository from '../database/repositories/eagleEyeContentR import { LoggingBase } from './loggingBase' import { EagleEyeContent, EagleEyeAction } from '../types/eagleEyeTypes' import { PageData, QueryData } from '../types/common' -import SequelizeRepository from '../database/repositories/sequelizeRepository' +import Error400 from '../errors/Error400' export interface EagleEyeContentUpsertData extends EagleEyeAction { content: EagleEyeContent @@ -25,36 +25,23 @@ export default class EagleEyeContentService extends LoggingBase { */ async upsert(data: EagleEyeContent): Promise { - const transaction = await SequelizeRepository.createTransaction(this.options) - - try { - - if (!data.url) { - throw new Error(`Can't upsert without url`) - } - - // find by url - const existing = await EagleEyeContentRepository.findByUrl(data.url, this.options) - - let record - - if (existing) { - record = await EagleEyeContentRepository.update(existing.id, data, this.options) - } else { - record = await EagleEyeContentRepository.create(data, this.options) - } - - await SequelizeRepository.commitTransaction(transaction) + if (!data.url) { + throw new Error400(this.options.language, 'errors.eagleEye.urlRequiredWhenUpserting') + } - return record - } catch (error) { - await SequelizeRepository.rollbackTransaction(transaction) + // find by url + const existing = await EagleEyeContentRepository.findByUrl(data.url, this.options) - SequelizeRepository.handleUniqueFieldError(error, this.options.language, 'EagleEyeContent') + let record - throw error + if (existing) { + record = await EagleEyeContentRepository.update(existing.id, data, this.options) + } else { + record = await EagleEyeContentRepository.create(data, this.options) } + return record + } async findById(id: string): Promise { From e8946abc9ae454f27421daf40b1ce71e98ea57b5 Mon Sep 17 00:00:00 2001 From: anilb Date: Sun, 5 Feb 2023 21:01:47 +0100 Subject: [PATCH 10/36] formatting, all routing finalized --- .../eagleEyeContent/eagleEyeActionCreate.ts | 11 ++ ...tentUpdate.ts => eagleEyeActionDestroy.ts} | 6 +- .../eagleEyeContent/eagleEyeContentList.ts | 17 --- backend/src/api/eagleEyeContent/index.ts | 10 +- backend/src/api/index.ts | 2 +- backend/src/database/models/eagleEyeAction.ts | 2 +- .../src/database/models/eagleEyeContent.ts | 3 +- .../eagleEyeActionRepository.test.ts | 12 +- .../eagleEyeContentRepository.test.ts | 55 +++++---- .../repositories/eagleEyeContentRepository.ts | 107 ++++++------------ backend/src/i18n/en.ts | 3 +- backend/src/security/permissions.ts | 10 ++ .../__tests__/eagleEyeContentService.test.ts | 15 +-- backend/src/services/eagleEyeActionService.ts | 96 +++++++++------- .../src/services/eagleEyeContentService.ts | 10 +- backend/src/types/eagleEyeTypes.ts | 36 +++--- 16 files changed, 190 insertions(+), 205 deletions(-) rename backend/src/api/eagleEyeContent/{eagleEyeContentUpdate.ts => eagleEyeActionDestroy.ts} (61%) delete mode 100644 backend/src/api/eagleEyeContent/eagleEyeContentList.ts diff --git a/backend/src/api/eagleEyeContent/eagleEyeActionCreate.ts b/backend/src/api/eagleEyeContent/eagleEyeActionCreate.ts index e69de29bb2..8d25a39212 100644 --- a/backend/src/api/eagleEyeContent/eagleEyeActionCreate.ts +++ b/backend/src/api/eagleEyeContent/eagleEyeActionCreate.ts @@ -0,0 +1,11 @@ +import Permissions from '../../security/permissions' +import EagleEyeActionService from '../../services/eagleEyeActionService' +import PermissionChecker from '../../services/user/permissionChecker' + +export default async (req, res) => { + new PermissionChecker(req).validateHas(Permissions.values.eagleEyeActionCreate) + + const payload = await new EagleEyeActionService(req).create(req.body, req.params.contentId) + + await req.responseHandler.success(req, res, payload) +} diff --git a/backend/src/api/eagleEyeContent/eagleEyeContentUpdate.ts b/backend/src/api/eagleEyeContent/eagleEyeActionDestroy.ts similarity index 61% rename from backend/src/api/eagleEyeContent/eagleEyeContentUpdate.ts rename to backend/src/api/eagleEyeContent/eagleEyeActionDestroy.ts index b1d3c0854e..7684b74174 100644 --- a/backend/src/api/eagleEyeContent/eagleEyeContentUpdate.ts +++ b/backend/src/api/eagleEyeContent/eagleEyeActionDestroy.ts @@ -1,11 +1,11 @@ import Permissions from '../../security/permissions' -import EagleEyeContentService from '../../services/eagleEyeContentService' +import EagleEyeActionService from '../../services/eagleEyeActionService' import PermissionChecker from '../../services/user/permissionChecker' export default async (req, res) => { - new PermissionChecker(req).validateHas(Permissions.values.eagleEyeContentEdit) + new PermissionChecker(req).validateHas(Permissions.values.eagleEyeActionDestroy) - const payload = await new EagleEyeContentService(req).update(req.params.id, req.body) + const payload = await new EagleEyeActionService(req).destroy(req.params.actionId) await req.responseHandler.success(req, res, payload) } diff --git a/backend/src/api/eagleEyeContent/eagleEyeContentList.ts b/backend/src/api/eagleEyeContent/eagleEyeContentList.ts deleted file mode 100644 index 40fd0fb9c9..0000000000 --- a/backend/src/api/eagleEyeContent/eagleEyeContentList.ts +++ /dev/null @@ -1,17 +0,0 @@ -import Permissions from '../../security/permissions' -import track from '../../segment/track' -import EagleEyeContentService from '../../services/eagleEyeContentService' -import PermissionChecker from '../../services/user/permissionChecker' - -export default async (req, res) => { - new PermissionChecker(req).validateHas(Permissions.values.eagleEyeContentRead) - const payload = await new EagleEyeContentService(req).findAndCountAll(req.query) - - if (req.query.filter && Object.keys(req.query.filter).length > 0) { - const platforms = req.query.filter.platforms ? req.query.filter.platforms.split(',') : [] - const nDays = req.query.filter.nDays - track('Eagle Eye Filter', { filter: req.query.filter, platforms, nDays }, { ...req }) - } - - await req.responseHandler.success(req, res, payload) -} diff --git a/backend/src/api/eagleEyeContent/index.ts b/backend/src/api/eagleEyeContent/index.ts index 5cd3220e19..167c11ab32 100644 --- a/backend/src/api/eagleEyeContent/index.ts +++ b/backend/src/api/eagleEyeContent/index.ts @@ -9,11 +9,7 @@ export default (app) => { `/tenant/:tenantId/eagleEyeContent/query`, safeWrap(require('./eagleEyeContentQuery').default), ) - app.put( - `/tenant/:tenantId/eagleEyeContent/:id`, - safeWrap(require('./eagleEyeContentUpdate').default), - ) - app.get(`/tenant/:tenantId/eagleEyeContent`, safeWrap(require('./eagleEyeContentList').default)) + app.get( `/tenant/:tenantId/eagleEyeContent/:id`, safeWrap(require('./eagleEyeContentFind').default), @@ -24,4 +20,8 @@ export default (app) => { safeWrap(require('./eagleEyeActionCreate').default), ) + app.delete( + `/tenant/:tenantId/eagleEyeContent/:contentId/action/:actionId`, + safeWrap(require('./eagleEyeActionDestroy').default), + ) } diff --git a/backend/src/api/index.ts b/backend/src/api/index.ts index 79fb63df77..de5e1c9e78 100644 --- a/backend/src/api/index.ts +++ b/backend/src/api/index.ts @@ -174,4 +174,4 @@ setImmediate(async () => { app.use(io.expressErrorHandler()) }) -export default server \ No newline at end of file +export default server diff --git a/backend/src/database/models/eagleEyeAction.ts b/backend/src/database/models/eagleEyeAction.ts index 14eaab4260..116b069fb2 100644 --- a/backend/src/database/models/eagleEyeAction.ts +++ b/backend/src/database/models/eagleEyeAction.ts @@ -38,7 +38,7 @@ export default (sequelize) => { }) models.eagleEyeAction.belongsTo(models.eagleEyeContent, { - as: 'content', + as: 'content', }) } diff --git a/backend/src/database/models/eagleEyeContent.ts b/backend/src/database/models/eagleEyeContent.ts index 198f28e754..9b00bc070c 100644 --- a/backend/src/database/models/eagleEyeContent.ts +++ b/backend/src/database/models/eagleEyeContent.ts @@ -15,7 +15,7 @@ const eagleEyeContentModel = { }, post: { type: DataTypes.JSONB, - allowNull: false + allowNull: false, }, url: { type: DataTypes.TEXT, @@ -47,7 +47,6 @@ export default (sequelize) => { as: 'actions', foreignKey: 'contentId', }) - } return eagleEyeContent diff --git a/backend/src/database/repositories/__tests__/eagleEyeActionRepository.test.ts b/backend/src/database/repositories/__tests__/eagleEyeActionRepository.test.ts index f3e035cc20..f3a64fa6ec 100644 --- a/backend/src/database/repositories/__tests__/eagleEyeActionRepository.test.ts +++ b/backend/src/database/repositories/__tests__/eagleEyeActionRepository.test.ts @@ -33,13 +33,16 @@ describe('eagleEyeActionRepository tests', () => { const contentCreated = await EagleEyeContentRepository.create(content, mockIRepositoryOptions) - - const action: EagleEyeAction= { + const action: EagleEyeAction = { type: EagleEyeActionType.BOOKMARK, timestamp: '2022-07-27T19:13:30Z', - } + } - const actionCreated = await EagleEyeActionRepository.createActionForContent(action, contentCreated.id, mockIRepositoryOptions) + const actionCreated = await EagleEyeActionRepository.createActionForContent( + action, + contentCreated.id, + mockIRepositoryOptions, + ) actionCreated.createdAt = (actionCreated.createdAt as Date).toISOString().split('T')[0] actionCreated.updatedAt = (actionCreated.updatedAt as Date).toISOString().split('T')[0] @@ -56,6 +59,5 @@ describe('eagleEyeActionRepository tests', () => { } expect(expectedAction).toStrictEqual(actionCreated) }) - }) }) diff --git a/backend/src/database/repositories/__tests__/eagleEyeContentRepository.test.ts b/backend/src/database/repositories/__tests__/eagleEyeContentRepository.test.ts index eef92a3679..96666a7148 100644 --- a/backend/src/database/repositories/__tests__/eagleEyeContentRepository.test.ts +++ b/backend/src/database/repositories/__tests__/eagleEyeContentRepository.test.ts @@ -3,8 +3,6 @@ import SequelizeTestUtils from '../../utils/sequelizeTestUtils' import { EagleEyeActionType, EagleEyeContent } from '../../../types/eagleEyeTypes' import EagleEyeActionRepository from '../eagleEyeActionRepository' - - const db = null describe('eagleEyeContentRepository tests', () => { @@ -422,17 +420,17 @@ describe('eagleEyeContentRepository tests', () => { describe('findAndCountAll method', () => { it('Should find eagle eye contant, various cases', async () => { - // create random tenant with one user + // create random tenant with one user const mockIRepositoryOptions = await SequelizeTestUtils.getTestIRepositoryOptions(db) // create additional users for same tenant to test out actionBy filtering const randomUser = await SequelizeTestUtils.getRandomUser() - console.log("random user: ") + console.log('random user: ') console.log(randomUser) const user2 = await mockIRepositoryOptions.database.user.create(randomUser) - + await mockIRepositoryOptions.database.tenantUser.create({ roles: ['admin'], status: 'active', @@ -456,7 +454,6 @@ describe('eagleEyeContentRepository tests', () => { mockIRepositoryOptions, ) - // one with a bookmark action let c2 = await EagleEyeContentRepository.create( { @@ -486,7 +483,7 @@ describe('eagleEyeContentRepository tests', () => { // another content with a thumbs-up(user1) and a bookmark(user2) action let c3 = await EagleEyeContentRepository.create( - { + { platform: 'devto', url: 'https://some-devto-url', post: { @@ -498,7 +495,7 @@ describe('eagleEyeContentRepository tests', () => { }, mockIRepositoryOptions, ) - + // add the thumbs up action await EagleEyeActionRepository.createActionForContent( { @@ -516,38 +513,40 @@ describe('eagleEyeContentRepository tests', () => { timestamp: '2022-09-30T23:11:10Z', }, c3.id, - {...mockIRepositoryOptions, currentUser: user2}, + { ...mockIRepositoryOptions, currentUser: user2 }, ) c3 = await EagleEyeContentRepository.findById(c3.id, mockIRepositoryOptions) - // filter by action type - let res = await EagleEyeContentRepository.findAndCountAll({ - advancedFilter: { - action: { - type: EagleEyeActionType.BOOKMARK - - } - } - }, mockIRepositoryOptions) - + let res = await EagleEyeContentRepository.findAndCountAll( + { + advancedFilter: { + action: { + type: EagleEyeActionType.BOOKMARK, + }, + }, + }, + mockIRepositoryOptions, + ) expect(res.count).toBe(2) - expect(res.rows.sort((a, b) => (a.createdAt > b.createdAt ? 1 : -1))).toStrictEqual([c2,c3]) + expect(res.rows.sort((a, b) => (a.createdAt > b.createdAt ? 1 : -1))).toStrictEqual([c2, c3]) // filter by actionBy - res = await EagleEyeContentRepository.findAndCountAll({ - advancedFilter: { - action: { - actionById: user2.id - } - } - }, mockIRepositoryOptions) + res = await EagleEyeContentRepository.findAndCountAll( + { + advancedFilter: { + action: { + actionById: user2.id, + }, + }, + }, + mockIRepositoryOptions, + ) expect(res.count).toBe(1) expect(res.rows).toStrictEqual([c3]) - }) }) }) diff --git a/backend/src/database/repositories/eagleEyeContentRepository.ts b/backend/src/database/repositories/eagleEyeContentRepository.ts index 66b87609f6..9a34d4d9d4 100644 --- a/backend/src/database/repositories/eagleEyeContentRepository.ts +++ b/backend/src/database/repositories/eagleEyeContentRepository.ts @@ -9,33 +9,26 @@ import { QueryOutput } from './filters/queryTypes' import EagleEyeActionRepository from './eagleEyeActionRepository' export default class EagleEyeContentRepository { - - static async create(data:EagleEyeContent, options:IRepositoryOptions): Promise{ + static async create( + data: EagleEyeContent, + options: IRepositoryOptions, + ): Promise { const currentTenant = SequelizeRepository.getCurrentTenant(options) - const record = await options.database.eagleEyeContent.create( - { - ...lodash.pick(data, [ - 'platform', - 'post', - 'url', - 'postedAt' - ]), - tenantId: currentTenant.id, - }, - ) + const record = await options.database.eagleEyeContent.create({ + ...lodash.pick(data, ['platform', 'post', 'url', 'postedAt']), + tenantId: currentTenant.id, + }) - if (data.actions){ - for (const action of data.actions){ + if (data.actions) { + for (const action of data.actions) { await EagleEyeActionRepository.createActionForContent(action, record.id, options) } } return this.findById(record.id, options) - } - static async update(id, data, options: IRepositoryOptions) { const currentUser = SequelizeRepository.getCurrentUser(options) @@ -55,15 +48,9 @@ export default class EagleEyeContentRepository { throw new Error404() } - record = await record.update( { - ...lodash.pick(data, [ - 'platform', - 'post', - 'postedAt', - 'url', - ]), + ...lodash.pick(data, ['platform', 'post', 'postedAt', 'url']), updatedById: currentUser.id, }, { @@ -74,13 +61,15 @@ export default class EagleEyeContentRepository { return this.findById(record.id, options) } - static async findById(id:string, options: IRepositoryOptions) { + static async findById(id: string, options: IRepositoryOptions) { const transaction = SequelizeRepository.getTransaction(options) - const include = [{ - model: options.database.eagleEyeAction, - as: 'actions', - }] + const include = [ + { + model: options.database.eagleEyeAction, + as: 'actions', + }, + ] const currentTenant = SequelizeRepository.getCurrentTenant(options) @@ -122,30 +111,17 @@ export default class EagleEyeContentRepository { } static async findAndCountAll( - { - advancedFilter = null as any, - limit = 0, - offset = 0, - orderBy = '', - }, + { advancedFilter = null as any, limit = 0, offset = 0, orderBy = '' }, options: IRepositoryOptions, ) { - const actionsSequelizeInclude = { model: options.database.eagleEyeAction, as: 'actions', where: {}, - } - if (advancedFilter && advancedFilter.action) { - - const actionQueryParser = new QueryParser( - { - }, - options, - ) + const actionQueryParser = new QueryParser({}, options) const parsedActionQuery: QueryOutput = actionQueryParser.parse({ filter: advancedFilter.action, @@ -156,15 +132,9 @@ export default class EagleEyeContentRepository { delete advancedFilter.action } - const include = [ - actionsSequelizeInclude, - ] - - const contentParser = new QueryParser( - {}, - options, - ) + const include = [actionsSequelizeInclude] + const contentParser = new QueryParser({}, options) const parsed: QueryOutput = contentParser.parse({ filter: advancedFilter, @@ -173,7 +143,6 @@ export default class EagleEyeContentRepository { offset, }) - let { rows, count, // eslint-disable-line prefer-const @@ -184,34 +153,32 @@ export default class EagleEyeContentRepository { limit: parsed.limit, offset: parsed.offset, transaction: SequelizeRepository.getTransaction(options), - subQuery: false + subQuery: false, }) // If we have an actions filter, we should query again to eager - // load the all actions on a content because previous query will + // load the all actions on a content because previous query will // omit actions that don't match the given action filter - if (Object.keys(actionsSequelizeInclude.where).length !== 0) - { - rows = (await options.database.eagleEyeContent.findAndCountAll({ - include: [{...actionsSequelizeInclude, where: {}}], - where: { id: { [Op.in]: rows.map( (i) => i.id) } }, - order: parsed.order, - limit: parsed.limit, - offset: parsed.offset, - transaction: SequelizeRepository.getTransaction(options), - subQuery: false - - })).rows + if (Object.keys(actionsSequelizeInclude.where).length !== 0) { + rows = ( + await options.database.eagleEyeContent.findAndCountAll({ + include: [{ ...actionsSequelizeInclude, where: {} }], + where: { id: { [Op.in]: rows.map((i) => i.id) } }, + order: parsed.order, + limit: parsed.limit, + offset: parsed.offset, + transaction: SequelizeRepository.getTransaction(options), + subQuery: false, + }) + ).rows } - rows = await this._populateRelationsForRows(rows) return { rows, count, limit: parsed.limit, offset: parsed.offset } - } - static async findByUrl(url:string, options: IRepositoryOptions) { + static async findByUrl(url: string, options: IRepositoryOptions) { const transaction = SequelizeRepository.getTransaction(options) const include = [] diff --git a/backend/src/i18n/en.ts b/backend/src/i18n/en.ts index 0c4a89b705..82896eb36d 100644 --- a/backend/src/i18n/en.ts +++ b/backend/src/i18n/en.ts @@ -109,7 +109,8 @@ const en = { message: 'Project not found', }, eagleEye: { - urlRequiredWhenUpserting: 'URL field is mandatory when upserting eagleEyeContent' + urlRequiredWhenUpserting: 'URL field is mandatory when upserting eagleEyeContent', + contentNotFound: 'Eagle eye content not found. Action will not be created.', }, integrations: { badEndpoint: 'Bad endpoint: {0}', diff --git a/backend/src/security/permissions.ts b/backend/src/security/permissions.ts index 74482a3f03..ec28c71024 100644 --- a/backend/src/security/permissions.ts +++ b/backend/src/security/permissions.ts @@ -422,6 +422,16 @@ class Permissions { allowedRoles: [roles.admin, roles.readonly], allowedPlans: [plans.essential, plans.growth], }, + eagleEyeActionCreate: { + id: 'eagleEyeActionCreate', + allowedRoles: [roles.admin], + allowedPlans: [plans.growth], + }, + eagleEyeActionDestroy: { + id: 'eagleEyeActionDestroy', + allowedRoles: [roles.admin], + allowedPlans: [plans.growth], + }, eagleEyeContentCreate: { id: 'eagleEyeContentCreate', allowedRoles: [roles.admin], diff --git a/backend/src/services/__tests__/eagleEyeContentService.test.ts b/backend/src/services/__tests__/eagleEyeContentService.test.ts index f67db5118e..9170b4a645 100644 --- a/backend/src/services/__tests__/eagleEyeContentService.test.ts +++ b/backend/src/services/__tests__/eagleEyeContentService.test.ts @@ -27,10 +27,12 @@ describe('EagleEyeContentService tests', () => { }, postedAt: '2020-05-27T15:13:30Z', tenantId: mockIRepositoryOptions.currentTenant.id, - actions: [{ - type: EagleEyeActionType.BOOKMARK, - timestamp: '2022-06-27T14:13:30Z', - }] + actions: [ + { + type: EagleEyeActionType.BOOKMARK, + timestamp: '2022-06-27T14:13:30Z', + }, + ], } const service = new EagleEyeContentService(mockIRepositoryOptions) @@ -60,11 +62,6 @@ describe('EagleEyeContentService tests', () => { expect(contents.rows).toStrictEqual([c1Upserted]) expect(c1Upserted.id).toEqual(c1.id) expect(contents.rows[0].post).toStrictEqual(contentWithSameUrl.post) - }) - - }) - - }) diff --git a/backend/src/services/eagleEyeActionService.ts b/backend/src/services/eagleEyeActionService.ts index 7711915543..75b71c9178 100644 --- a/backend/src/services/eagleEyeActionService.ts +++ b/backend/src/services/eagleEyeActionService.ts @@ -1,5 +1,6 @@ import EagleEyeActionRepository from '../database/repositories/eagleEyeActionRepository' import EagleEyeContentRepository from '../database/repositories/eagleEyeContentRepository' +import SequelizeRepository from '../database/repositories/sequelizeRepository' import Error404 from '../errors/Error404' import { EagleEyeAction, EagleEyeActionType } from '../types/eagleEyeTypes' import { IServiceOptions } from './IServiceOptions' @@ -14,54 +15,71 @@ export default class EagleEyeActionService extends LoggingBase { } async create(data: EagleEyeAction, contentId: string): Promise { + const transaction = await SequelizeRepository.createTransaction(this.options) + // find content const content = await EagleEyeContentRepository.findById(contentId, this.options) if (!content) { - throw new Error404('Content not found..') + throw new Error404(this.options.language, 'errors.eagleEye.contentNotFound') } - const existingUserActions: EagleEyeAction[] = content.actions - .filter((a) => a.actionById === this.options.currentUser.id) + const existingUserActions: EagleEyeAction[] = content.actions.filter( + (a) => a.actionById === this.options.currentUser.id, + ) const existingUserActionTypes = existingUserActions.map((a) => a.type) - // check if already bookmarked - if yes ignore the new action and return existing - if ( - data.type === EagleEyeActionType.BOOKMARK && - existingUserActionTypes.includes(EagleEyeActionType.BOOKMARK) - ) { - return existingUserActions.find( (a) => a.type === EagleEyeActionType.BOOKMARK) - } - - // thumbs up and down should be mutually exclusive - if ( - data.type === EagleEyeActionType.THUMBS_DOWN && - existingUserActionTypes.includes(EagleEyeActionType.THUMBS_UP) - ) { - await EagleEyeActionRepository.removeActionFromContent( - EagleEyeActionType.THUMBS_UP, - contentId, - this.options, - ) - } else if ( - data.type === EagleEyeActionType.THUMBS_UP && - existingUserActionTypes.includes(EagleEyeActionType.THUMBS_DOWN) - ) { - await EagleEyeActionRepository.removeActionFromContent( - EagleEyeActionType.THUMBS_DOWN, - contentId, - this.options, - ) - } - - // add new action - return EagleEyeActionRepository.createActionForContent(data, contentId, this.options) + try { + // check if already bookmarked - if yes ignore the new action and return existing + if ( + data.type === EagleEyeActionType.BOOKMARK && + existingUserActionTypes.includes(EagleEyeActionType.BOOKMARK) + ) { + return existingUserActions.find((a) => a.type === EagleEyeActionType.BOOKMARK) + } + + // thumbs up and down should be mutually exclusive + if ( + data.type === EagleEyeActionType.THUMBS_DOWN && + existingUserActionTypes.includes(EagleEyeActionType.THUMBS_UP) + ) { + await EagleEyeActionRepository.removeActionFromContent( + EagleEyeActionType.THUMBS_UP, + contentId, + this.options, + ) + } else if ( + data.type === EagleEyeActionType.THUMBS_UP && + existingUserActionTypes.includes(EagleEyeActionType.THUMBS_DOWN) + ) { + await EagleEyeActionRepository.removeActionFromContent( + EagleEyeActionType.THUMBS_DOWN, + contentId, + this.options, + ) + } + + // add new action + const record = await EagleEyeActionRepository.createActionForContent( + data, + contentId, + this.options, + ) + + await SequelizeRepository.commitTransaction(transaction) + + return record + } catch (error) { + await SequelizeRepository.rollbackTransaction(transaction) + + SequelizeRepository.handleUniqueFieldError(error, this.options.language, 'EagleEyeContent') + + throw error + } } - - async destroy(id: string){ - + async destroy(id: string) { const action = await EagleEyeActionRepository.findById(id, this.options) const contentId = action.contentId @@ -72,8 +90,8 @@ export default class EagleEyeActionService extends LoggingBase { const content = await EagleEyeContentRepository.findById(contentId, this.options) // if content no longer has any actions attached to it, also delete the content - if (content.actions.length === 0){ - await EagleEyeContentRepository.destroy(contentId, this.options) + if (content.actions.length === 0) { + await EagleEyeContentRepository.destroy(contentId, this.options) } } } diff --git a/backend/src/services/eagleEyeContentService.ts b/backend/src/services/eagleEyeContentService.ts index fdbd133d55..4a58b4f766 100644 --- a/backend/src/services/eagleEyeContentService.ts +++ b/backend/src/services/eagleEyeContentService.ts @@ -24,7 +24,6 @@ export default class EagleEyeContentService extends LoggingBase { * @returns Created EagleEyeContent record. */ async upsert(data: EagleEyeContent): Promise { - if (!data.url) { throw new Error400(this.options.language, 'errors.eagleEye.urlRequiredWhenUpserting') } @@ -41,7 +40,6 @@ export default class EagleEyeContentService extends LoggingBase { } return record - } async findById(id: string): Promise { @@ -59,9 +57,11 @@ export default class EagleEyeContentService extends LoggingBase { ) } - /** TODO + /** + TODO + */ + /* eslint-disable-next-line */ async search(args) { - + return null } - */ } diff --git a/backend/src/types/eagleEyeTypes.ts b/backend/src/types/eagleEyeTypes.ts index 556ed1fd8c..2aa5a9bd1d 100644 --- a/backend/src/types/eagleEyeTypes.ts +++ b/backend/src/types/eagleEyeTypes.ts @@ -1,27 +1,25 @@ export enum EagleEyeActionType { - THUMBS_UP = 'thumbs-up', - THUMBS_DOWN = 'thumbs-down', - BOOKMARK = 'bookmark', + THUMBS_UP = 'thumbs-up', + THUMBS_DOWN = 'thumbs-down', + BOOKMARK = 'bookmark', } - export interface EagleEyeAction { - id?: string - type: EagleEyeActionType - timestamp: Date | string - createdAt?: Date | string - updatedAt?: Date | string + id?: string + type: EagleEyeActionType + timestamp: Date | string + createdAt?: Date | string + updatedAt?: Date | string } export interface EagleEyeContent { - id?: string - platform: string - post: any - url: string - actions?: EagleEyeAction[] - tenantId: string - postedAt: string - createdAt?: Date | string - updatedAt?: Date | string + id?: string + platform: string + post: any + url: string + actions?: EagleEyeAction[] + tenantId: string + postedAt: string + createdAt?: Date | string + updatedAt?: Date | string } - From 87a7a41bd5efbe96931236eb661b01cdc1f22f7f Mon Sep 17 00:00:00 2001 From: anilb Date: Sun, 5 Feb 2023 21:07:17 +0100 Subject: [PATCH 11/36] removed un-used get posts by keywords file --- .../usecases/hackerNews/getPostsByKeywords.ts | 27 ------------------- 1 file changed, 27 deletions(-) delete mode 100644 backend/src/serverless/integrations/usecases/hackerNews/getPostsByKeywords.ts diff --git a/backend/src/serverless/integrations/usecases/hackerNews/getPostsByKeywords.ts b/backend/src/serverless/integrations/usecases/hackerNews/getPostsByKeywords.ts deleted file mode 100644 index 0e19a3ac5e..0000000000 --- a/backend/src/serverless/integrations/usecases/hackerNews/getPostsByKeywords.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { IServiceOptions } from '../../../../services/IServiceOptions' -import { EagleEyeResponses, EagleEyeInput } from '../../types/hackerNewsTypes' -import { Logger } from '../../../../utils/logging' -import { timeout } from '../../../../utils/timing' -import EagleEyeContentService from '../../../../services/eagleEyeContentService' - -async function getPostsByKeyword( - input: EagleEyeInput, - options: IServiceOptions, - logger: Logger, -): Promise { - await timeout(2000) - - try { - const eagleEyeService = new EagleEyeContentService(options) - return await eagleEyeService.keywordMatch({ - keywords: input.keywords, - nDays: input.nDays, - platform: 'hacker_news', - }) - } catch (err) { - logger.error({ err, input }, 'Error while getting posts by keyword in EagleEye') - throw err - } -} - -export default getPostsByKeyword From 9452bbfd7cec8028584aa430122220a3c4359d7f Mon Sep 17 00:00:00 2001 From: anilb Date: Sun, 5 Feb 2023 21:33:54 +0100 Subject: [PATCH 12/36] getPostsByKeyword re-added bcs it's being used by hackernews --- .../usecases/hackerNews/getPostsByKeywords.ts | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 backend/src/serverless/integrations/usecases/hackerNews/getPostsByKeywords.ts diff --git a/backend/src/serverless/integrations/usecases/hackerNews/getPostsByKeywords.ts b/backend/src/serverless/integrations/usecases/hackerNews/getPostsByKeywords.ts new file mode 100644 index 0000000000..7d5a837546 --- /dev/null +++ b/backend/src/serverless/integrations/usecases/hackerNews/getPostsByKeywords.ts @@ -0,0 +1,27 @@ +import { IServiceOptions } from '../../../../services/IServiceOptions' +import { EagleEyeResponses, EagleEyeInput } from '../../types/hackerNewsTypes' +import { Logger } from '../../../../utils/logging' +import { timeout } from '../../../../utils/timing' +import EagleEyeContentService from '../../../../services/eagleEyeContentService' + +async function getPostsByKeyword( + input: EagleEyeInput, + options: IServiceOptions, + logger: Logger, +): Promise { + await timeout(2000) + + try { + const eagleEyeService = new EagleEyeContentService(options) + return await eagleEyeService.keywordMatch({ + keywords: input.keywords, + nDays: input.nDays, + platform: 'hacker_news', + }) + } catch (err) { + logger.error({ err, input }, 'Error while getting posts by keyword in EagleEye') + throw err + } +} + +export default getPostsByKeyword \ No newline at end of file From cb967c00d57a3c53bacaa0fb79b11fd8970b8afb Mon Sep 17 00:00:00 2001 From: Joan Reyero Date: Tue, 7 Feb 2023 13:36:17 +0100 Subject: [PATCH 13/36] Added endpoint for EagleEye settings --- .../eagleEyeContent/eagleEyeSettingsUpdate.ts | 11 ++ backend/src/api/eagleEyeContent/index.ts | 5 + .../U1675702339__eagleEyeSettings.sql | 2 + .../V1675702339__eagleEyeSettings.sql | 2 + backend/src/database/models/user.ts | 7 + .../database/repositories/userRepository.ts | 31 ++++ backend/src/i18n/en.ts | 12 ++ .../src/services/eagleEyeSettingsService.ts | 144 ++++++++++++++++++ backend/src/types/eagleEyeTypes.ts | 47 ++++++ 9 files changed, 261 insertions(+) create mode 100644 backend/src/api/eagleEyeContent/eagleEyeSettingsUpdate.ts create mode 100644 backend/src/database/migrations/U1675702339__eagleEyeSettings.sql create mode 100644 backend/src/database/migrations/V1675702339__eagleEyeSettings.sql create mode 100644 backend/src/services/eagleEyeSettingsService.ts diff --git a/backend/src/api/eagleEyeContent/eagleEyeSettingsUpdate.ts b/backend/src/api/eagleEyeContent/eagleEyeSettingsUpdate.ts new file mode 100644 index 0000000000..635e4a7411 --- /dev/null +++ b/backend/src/api/eagleEyeContent/eagleEyeSettingsUpdate.ts @@ -0,0 +1,11 @@ +import Permissions from '../../security/permissions' +import EagleEyeSettingsService from '../../services/eagleEyeSettingsService' +import PermissionChecker from '../../services/user/permissionChecker' + +export default async (req, res) => { + new PermissionChecker(req).validateHas(Permissions.values.eagleEyeActionCreate) + + const payload = await new EagleEyeSettingsService(req).update(req.body) + + await req.responseHandler.success(req, res, payload) +} diff --git a/backend/src/api/eagleEyeContent/index.ts b/backend/src/api/eagleEyeContent/index.ts index 167c11ab32..cbc3f21db4 100644 --- a/backend/src/api/eagleEyeContent/index.ts +++ b/backend/src/api/eagleEyeContent/index.ts @@ -20,6 +20,11 @@ export default (app) => { safeWrap(require('./eagleEyeActionCreate').default), ) + app.put( + `/tenant/:tenantId/eagleEyeContent/settings`, + safeWrap(require('./eagleEyeSettingsUpdate').default), + ) + app.delete( `/tenant/:tenantId/eagleEyeContent/:contentId/action/:actionId`, safeWrap(require('./eagleEyeActionDestroy').default), diff --git a/backend/src/database/migrations/U1675702339__eagleEyeSettings.sql b/backend/src/database/migrations/U1675702339__eagleEyeSettings.sql new file mode 100644 index 0000000000..e3a0f4fe1e --- /dev/null +++ b/backend/src/database/migrations/U1675702339__eagleEyeSettings.sql @@ -0,0 +1,2 @@ +ALTER TABLE public."users" +DROP COLUMN "eagleEyeSettings"; \ No newline at end of file diff --git a/backend/src/database/migrations/V1675702339__eagleEyeSettings.sql b/backend/src/database/migrations/V1675702339__eagleEyeSettings.sql new file mode 100644 index 0000000000..7e8e46fbf2 --- /dev/null +++ b/backend/src/database/migrations/V1675702339__eagleEyeSettings.sql @@ -0,0 +1,2 @@ +ALTER TABLE public."users" +ADD COLUMN "eagleEyeSettings" JSONB DEFAULT '{"onboarded": false}'; diff --git a/backend/src/database/models/user.ts b/backend/src/database/models/user.ts index a64482691b..3e9990465e 100644 --- a/backend/src/database/models/user.ts +++ b/backend/src/database/models/user.ts @@ -102,6 +102,13 @@ export default (sequelize, DataTypes) => { len: [0, 255], }, }, + eagleEyeSettings: { + type: DataTypes.JSONB, + allowNull: false, + defaultValue: { + onboarded: false, + }, + }, }, { indexes: [ diff --git a/backend/src/database/repositories/userRepository.ts b/backend/src/database/repositories/userRepository.ts index 8b80aee5ed..781e93d1cf 100644 --- a/backend/src/database/repositories/userRepository.ts +++ b/backend/src/database/repositories/userRepository.ts @@ -320,6 +320,37 @@ export default class UserRepository { return this.findById(user.id, options) } + static async updateEagleEyeSettings(id: string, data, options: IRepositoryOptions) { + const currentUser = SequelizeRepository.getCurrentUser(options) + const transaction = SequelizeRepository.getTransaction(options) + + const user = await options.database.user.findByPk(id, { + transaction, + }) + + await user.update( + { + eagleEyeSettings: data, + updatedById: currentUser.id, + }, + { transaction }, + ) + + await AuditLogRepository.log( + { + entityName: 'user', + entityId: user.id, + action: AuditLogRepository.UPDATE, + values: { + ...user.get({ plain: true }), + eagleEyeSettings: data, + }, + }, + options, + ) + return this.findById(user.id, options) + } + static async findByEmail(email, options: IRepositoryOptions) { const transaction = SequelizeRepository.getTransaction(options) diff --git a/backend/src/i18n/en.ts b/backend/src/i18n/en.ts index 82896eb36d..f0fad60032 100644 --- a/backend/src/i18n/en.ts +++ b/backend/src/i18n/en.ts @@ -111,6 +111,18 @@ const en = { eagleEye: { urlRequiredWhenUpserting: 'URL field is mandatory when upserting eagleEyeContent', contentNotFound: 'Eagle eye content not found. Action will not be created.', + feedSettingsMissing: 'Feed settings are missing. Settings not updated.', + keywordsMissing: + 'Either keywords or exactKeywords are required in feeds. Settings not updated.', + platformMissing: + 'feed.platforms is required and must be a non-empty list. Settings not updated.', + platformInvalid: `feed.platforms contains {0}, which is not in [{1}]. Settings not updated.`, + publishedDateMissing: + 'feed.publishedDate is missing or invalid. It should be one of [{0}]. Settings not updated.', + emailInvalid: 'emailDigest.email needs a valid email address. Settings not updated.', + frequencyInvalid: + 'emailDigest.frequency needs to be one of daily, weekly. Settings not updated.', + timeInvalid: 'emailDigest.time needs to be a valid time. Settings not updated.', }, integrations: { badEndpoint: 'Bad endpoint: {0}', diff --git a/backend/src/services/eagleEyeSettingsService.ts b/backend/src/services/eagleEyeSettingsService.ts new file mode 100644 index 0000000000..cd6826cf29 --- /dev/null +++ b/backend/src/services/eagleEyeSettingsService.ts @@ -0,0 +1,144 @@ +import moment from 'moment' +import lodash from 'lodash' +import SequelizeRepository from '../database/repositories/sequelizeRepository' +import UserRepository from '../database/repositories/userRepository' +import Error400 from '../errors/Error400' +import { + EagleEyeSettings, + EagleEyeFeedSettings, + EagleEyePlatforms, + EagleEyePublishedDates, + EagleEyeEmailDigestSettings, +} from '../types/eagleEyeTypes' +import { IServiceOptions } from './IServiceOptions' +import { LoggingBase } from './loggingBase' + +export default class EagleEyeSettingsService extends LoggingBase { + options: IServiceOptions + + constructor(options) { + super(options) + this.options = options + } + + getFeed(data: EagleEyeFeedSettings) { + if (!data) { + throw new Error400(this.options.language, 'errors.eagleEye.feedSettingsMissing') + } + + if (!data.keywords && !data.exactKeywords) { + throw new Error400(this.options.language, 'errors.eagleEye.keywordsMissing') + } + + if (!data.platforms || data.platforms.length === 0) { + throw new Error400(this.options.language, 'errors.eagleEye.platformMissing') + } + + const platforms = Object.values(EagleEyePlatforms) as string[] + data.platforms.forEach((platform) => { + if (!platforms.includes(platform)) { + throw new Error400( + this.options.language, + 'errors.eagleEye.platformInvalid', + platform, + platforms.join(', '), + ) + } + }) + + const publishedDates = Object.values(EagleEyePublishedDates) as string[] + if (publishedDates.indexOf(data.publishedDate as string) === -1) { + throw new Error400( + this.options.language, + 'errors.eagleEye.publishedDateMissing', + publishedDates.join(', '), + ) + } + + data.publishedDate = EagleEyeSettingsService.switchDate(data.publishedDate as string) + return lodash.pick(data, [ + 'keywords', + 'exactKeywords', + 'excludedKeywords', + 'publishedDate', + 'platforms', + ]) + } + + static switchDate(date: string) { + switch (date) { + case 'Last 24h': + return moment().subtract(1, 'days').format('YYYY-MM-DD') + case 'Last 7 days': + return moment().subtract(7, 'days').format('YYYY-MM-DD') + case 'Last 14 days': + return moment().subtract(14, 'days').format('YYYY-MM-DD') + case 'Last 30 days': + return moment().subtract(30, 'days').format('YYYY-MM-DD') + case 'Last 90 days': + return moment().subtract(90, 'days').format('YYYY-MM-DD') + default: + return null + } + } + + getEmailDigestSettings(data: EagleEyeEmailDigestSettings, feed: EagleEyeFeedSettings) { + if (!data.matchFeedSettings) { + data.feed = this.getFeed(data.feed) + } else { + data.feed = feed + } + + const emailRegex = + /^(([^<>()[\]\\.,;:\s@"]+(\.[^<>()[\]\\.,;:\s@"]+)*)|(".+"))@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\])|(([a-zA-Z\-0-9]+\.)+[a-zA-Z]{2,}))$/ + if (!emailRegex.test(data.email)) { + throw new Error400(this.options.language, 'errors.eagleEye.emailInvalid') + } + + if (['daily', 'weekly'].indexOf(data.frequency) === -1) { + throw new Error400(this.options.language, 'errors.eagleEye.frequencyInvalid') + } + + const timeRegex = /^([0-1]?[0-9]|2[0-3]):[0-5][0-9]?$/ + if (!timeRegex.test(data.time)) { + throw new Error400(this.options.language, 'errors.eagleEye.timeInvalid') + } + + return lodash.pick(data, ['email', 'frequency', 'time', 'matchFeedSettings', 'feed']) + } + + async update(data: EagleEyeSettings): Promise { + const transaction = await SequelizeRepository.createTransaction(this.options) + try { + data.onboarded = true + + data.feed = this.getFeed(data.feed) + + if (data.emailDigest || data.emailDigestActive) { + data.emailDigestActive = true + + data.emailDigest = this.getEmailDigestSettings(data.emailDigest, data.feed) + } else { + data.emailDigestActive = false + } + + data = lodash.pick(data, ['onboarded', 'feed', 'emailDigestActive', 'emailDigest']) + + const userOut = await UserRepository.updateEagleEyeSettings( + this.options.currentUser.id, + { eagleEyeSettings: data }, + { ...this.options, transaction }, + ) + + await SequelizeRepository.commitTransaction(transaction) + + return userOut.eagleEyeSettings + } catch (error) { + await SequelizeRepository.rollbackTransaction(transaction) + + SequelizeRepository.handleUniqueFieldError(error, this.options.language, 'EagleEyeContent') + + throw error + } + } +} diff --git a/backend/src/types/eagleEyeTypes.ts b/backend/src/types/eagleEyeTypes.ts index 2aa5a9bd1d..2f1f2d4d59 100644 --- a/backend/src/types/eagleEyeTypes.ts +++ b/backend/src/types/eagleEyeTypes.ts @@ -23,3 +23,50 @@ export interface EagleEyeContent { createdAt?: Date | string updatedAt?: Date | string } + +export interface EagleEyeFeedSettings { + keywords: string[] + exactKeywords: string[] + excludedKeywords: string[] + publishedDate: string | Date + platforms: string[] +} + +export interface EagleEyeEmailDigestSettings { + email: string + frequency: 'daily' | 'weekly' + time: string + feed: EagleEyeFeedSettings + matchFeedSettings: boolean +} + +export interface EagleEyeSettings { + onboarded: boolean + feed: EagleEyeFeedSettings + emailDigestActive: boolean + emailDigest?: EagleEyeEmailDigestSettings +} + +// Enum for EagleEyePlatforms +export enum EagleEyePlatforms { + GITHUB = 'github', + HACKERNEWS = 'hackernews', + DEVTO = 'devto', + REDDIT = 'reddit', + MEDIUM = 'medium', + STACKOVERFLOW = 'stackoverflow', + TWITTER = 'twitter', + YOUTUBE = 'youtube', + PRODUCTHUNT = 'producthunt', + KAGGLE = 'kaggle', + HASHNODE = 'hashnode', + LINKEDIN = 'linkedin', +} + +export enum EagleEyePublishedDates { + LAST_24_HOURS = 'Last 24h', + LAST_7_DAYS = 'Last 7 days', + LAST_14_DAYS = 'Last 14 days', + LAST_30_DAYS = 'Last 30 days', + LAST_90_DAYS = 'Last 90 days', +} From 409efc12815297f05c0bd51c257a1ac82f9b7df9 Mon Sep 17 00:00:00 2001 From: Joan Reyero Date: Tue, 7 Feb 2023 13:55:19 +0100 Subject: [PATCH 14/36] Added comments --- .../usecases/hackerNews/getPostsByKeywords.ts | 2 +- .../src/services/eagleEyeSettingsService.ts | 42 ++++++++++++++++++- 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/backend/src/serverless/integrations/usecases/hackerNews/getPostsByKeywords.ts b/backend/src/serverless/integrations/usecases/hackerNews/getPostsByKeywords.ts index 7d5a837546..0e19a3ac5e 100644 --- a/backend/src/serverless/integrations/usecases/hackerNews/getPostsByKeywords.ts +++ b/backend/src/serverless/integrations/usecases/hackerNews/getPostsByKeywords.ts @@ -24,4 +24,4 @@ async function getPostsByKeyword( } } -export default getPostsByKeyword \ No newline at end of file +export default getPostsByKeyword diff --git a/backend/src/services/eagleEyeSettingsService.ts b/backend/src/services/eagleEyeSettingsService.ts index cd6826cf29..d500a8ff40 100644 --- a/backend/src/services/eagleEyeSettingsService.ts +++ b/backend/src/services/eagleEyeSettingsService.ts @@ -21,19 +21,28 @@ export default class EagleEyeSettingsService extends LoggingBase { this.options = options } + /** + * Validate and normalize feed settings. + * @param data Feed data of type EagleEyeFeedSettings + * @returns Normalized feed data if the input is valid. Otherwise a 400 Error + */ getFeed(data: EagleEyeFeedSettings) { + // Feed is compulsory if (!data) { throw new Error400(this.options.language, 'errors.eagleEye.feedSettingsMissing') } + // We need at least one of keywords or exactKeywords if (!data.keywords && !data.exactKeywords) { throw new Error400(this.options.language, 'errors.eagleEye.keywordsMissing') } + // We need at least one platform if (!data.platforms || data.platforms.length === 0) { throw new Error400(this.options.language, 'errors.eagleEye.platformMissing') } + // Make sure platforms are in the allowed list const platforms = Object.values(EagleEyePlatforms) as string[] data.platforms.forEach((platform) => { if (!platforms.includes(platform)) { @@ -46,6 +55,7 @@ export default class EagleEyeSettingsService extends LoggingBase { } }) + // We need a date. Make sure it's in the allowed list. const publishedDates = Object.values(EagleEyePublishedDates) as string[] if (publishedDates.indexOf(data.publishedDate as string) === -1) { throw new Error400( @@ -55,7 +65,10 @@ export default class EagleEyeSettingsService extends LoggingBase { ) } + // Convert the relative string date to a Date data.publishedDate = EagleEyeSettingsService.switchDate(data.publishedDate as string) + + // Remove any extra fields return lodash.pick(data, [ 'keywords', 'exactKeywords', @@ -65,6 +78,11 @@ export default class EagleEyeSettingsService extends LoggingBase { ]) } + /** + * Convert a relative string date to a Date. For example, 30 days ago -> 2020-01-01 + * @param date String date. Can be one of EagleEyePublishedDates + * @returns The corresponding Date + */ static switchDate(date: string) { switch (date) { case 'Last 24h': @@ -82,48 +100,70 @@ export default class EagleEyeSettingsService extends LoggingBase { } } + /** + * Validate and normalize email digest settings. + * @param data Email digest settings of type EagleEyeEmailDigestSettings + * @param feed Standard feed settings of type EagleEyeFeedSettings + * @returns The normalized email digest settings if the input is valid. Otherwise a 400 Error. + */ getEmailDigestSettings(data: EagleEyeEmailDigestSettings, feed: EagleEyeFeedSettings) { + // If the matchFeedSettings option is toggled, we set the email feed settings to the standard feed settings. + // Otherwise, we validate and normalize the email feed settings. if (!data.matchFeedSettings) { data.feed = this.getFeed(data.feed) } else { data.feed = feed } + // Make sure the email exists and is valid const emailRegex = /^(([^<>()[\]\\.,;:\s@"]+(\.[^<>()[\]\\.,;:\s@"]+)*)|(".+"))@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\])|(([a-zA-Z\-0-9]+\.)+[a-zA-Z]{2,}))$/ if (!emailRegex.test(data.email)) { throw new Error400(this.options.language, 'errors.eagleEye.emailInvalid') } + // Make sure the frequency exists and is valid if (['daily', 'weekly'].indexOf(data.frequency) === -1) { throw new Error400(this.options.language, 'errors.eagleEye.frequencyInvalid') } + // Make sure the time exists and is valid const timeRegex = /^([0-1]?[0-9]|2[0-3]):[0-5][0-9]?$/ if (!timeRegex.test(data.time)) { throw new Error400(this.options.language, 'errors.eagleEye.timeInvalid') } + // Remove any extra fields return lodash.pick(data, ['email', 'frequency', 'time', 'matchFeedSettings', 'feed']) } + /** + * Validate, normalize and update EagleEye settings. + * @param data Input of type EagleEyeSettings + * @returns Normalized EagleEyeSettings if the input is valid. Otherwise a 400 Error. + */ async update(data: EagleEyeSettings): Promise { const transaction = await SequelizeRepository.createTransaction(this.options) try { + // At this point onboarded is true always data.onboarded = true + // Validate and normalize feed settings data.feed = this.getFeed(data.feed) + // If an email digest was sent, validate and normalize email digest settings + // Otherwise, set email digest to false if (data.emailDigest || data.emailDigestActive) { data.emailDigestActive = true - data.emailDigest = this.getEmailDigestSettings(data.emailDigest, data.feed) } else { data.emailDigestActive = false } + // Remove any extra fields data = lodash.pick(data, ['onboarded', 'feed', 'emailDigestActive', 'emailDigest']) + // Update the user's EagleEye settings const userOut = await UserRepository.updateEagleEyeSettings( this.options.currentUser.id, { eagleEyeSettings: data }, From fb66be53977cf5ec76e84e30068425cd66f83f7a Mon Sep 17 00:00:00 2001 From: Joan Reyero Date: Tue, 7 Feb 2023 15:23:44 +0100 Subject: [PATCH 15/36] Fixed frontend sync --- backend/src/services/eagleEyeSettingsService.ts | 2 +- backend/src/types/eagleEyeTypes.ts | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/backend/src/services/eagleEyeSettingsService.ts b/backend/src/services/eagleEyeSettingsService.ts index d500a8ff40..011f5879fe 100644 --- a/backend/src/services/eagleEyeSettingsService.ts +++ b/backend/src/services/eagleEyeSettingsService.ts @@ -166,7 +166,7 @@ export default class EagleEyeSettingsService extends LoggingBase { // Update the user's EagleEye settings const userOut = await UserRepository.updateEagleEyeSettings( this.options.currentUser.id, - { eagleEyeSettings: data }, + data, { ...this.options, transaction }, ) diff --git a/backend/src/types/eagleEyeTypes.ts b/backend/src/types/eagleEyeTypes.ts index 2f1f2d4d59..91bc67883f 100644 --- a/backend/src/types/eagleEyeTypes.ts +++ b/backend/src/types/eagleEyeTypes.ts @@ -65,8 +65,8 @@ export enum EagleEyePlatforms { export enum EagleEyePublishedDates { LAST_24_HOURS = 'Last 24h', - LAST_7_DAYS = 'Last 7 days', - LAST_14_DAYS = 'Last 14 days', - LAST_30_DAYS = 'Last 30 days', - LAST_90_DAYS = 'Last 90 days', + LAST_7_DAYS = 'Last 7d', + LAST_14_DAYS = 'Last 14d', + LAST_30_DAYS = 'Last 30d', + LAST_90_DAYS = 'Last 90d', } From 061fd79c35711199a8ec41aed0093448660bdc44 Mon Sep 17 00:00:00 2001 From: anilb Date: Wed, 1 Feb 2023 16:15:00 +0100 Subject: [PATCH 16/36] eagle eye content v2 start --- .../U1675259471__eagleEyeActions.sql | 59 ++++++++++++++++++ .../V1675259471__eagleEyeActions.sql | 32 ++++++++++ backend/src/database/models/eagleEyeAction.ts | 62 +++++++++++++++++++ .../src/database/models/eagleEyeContent.ts | 36 +++++------ 4 files changed, 170 insertions(+), 19 deletions(-) create mode 100644 backend/src/database/migrations/U1675259471__eagleEyeActions.sql create mode 100644 backend/src/database/migrations/V1675259471__eagleEyeActions.sql create mode 100644 backend/src/database/models/eagleEyeAction.ts diff --git a/backend/src/database/migrations/U1675259471__eagleEyeActions.sql b/backend/src/database/migrations/U1675259471__eagleEyeActions.sql new file mode 100644 index 0000000000..30b8b2c037 --- /dev/null +++ b/backend/src/database/migrations/U1675259471__eagleEyeActions.sql @@ -0,0 +1,59 @@ +DROP TABLE IF EXISTS "eagleEyeContents"; + +DROP TYPE "eagleEyeContents_actions_type"; + +create table "eagleEyeContents"; +( + id uuid not null primary key, + "sourceId" text not null, + "vectorId" text not null, + status varchar(255) default NULL::character varying, + title text not null, + username text not null, + url text not null, + text text, + timestamp timestamp with time zone not null, + platform text not null, + keywords text [], + "similarityScore" double precision, + "userAttributes" jsonb, + "postAttributes" jsonb, + "importHash" varchar(255), + "createdAt" timestamp with time zone not null, + "updatedAt" timestamp with time zone not null, + "deletedAt" timestamp with time zone, + "tenantId" uuid not null references tenants on update cascade, + "createdById" uuid references users on update cascade on delete + set null, + "updatedById" uuid references users on update cascade on delete + set null, + "exactKeywords" text [] +); + +alter table "eagleEyeContents" owner to postgres; + +create index discord on "eagleEyeContents" ("vectorId", status); + +create index members_email_tenant_id on "eagleEyeContents" (id) +where ("deletedAt" IS NULL); + +create index members_joined_at_tenant_id on "eagleEyeContents" (id) +where ("deletedAt" IS NULL); + +create index members_username on "eagleEyeContents" using gin (id); + +create index slack on "eagleEyeContents" (id); + +create index twitter on "eagleEyeContents" (id); + +create unique index eagle_eye_contents_import_hash_tenant_id on "eagleEyeContents" ("importHash", "tenantId") +where ("deletedAt" IS NULL); + +create index eagle_eye_contents_platform_tenant_id_timestamp on "eagleEyeContents" (platform, "tenantId", timestamp) +where ("deletedAt" IS NULL); + +create index eagle_eye_contents_status_tenant_id_timestamp on "eagleEyeContents" (status, "tenantId", timestamp) +where ("deletedAt" IS NULL); + +create index eagle_eye_contents_tenant_id_timestamp on "eagleEyeContents" ("tenantId", timestamp) +where ("deletedAt" IS NULL); \ No newline at end of file diff --git a/backend/src/database/migrations/V1675259471__eagleEyeActions.sql b/backend/src/database/migrations/V1675259471__eagleEyeActions.sql new file mode 100644 index 0000000000..6ca96a3afb --- /dev/null +++ b/backend/src/database/migrations/V1675259471__eagleEyeActions.sql @@ -0,0 +1,32 @@ +DROP TABLE IF EXISTS "eagleEyeContents"; + +CREATE TABLE public."eageEyeContents" ( + "id" uuid NOT NULL, + "content" jsonb NOT NULL, + "tenantId" uuid NOT NULL, + "createdAt" timestamptz NOT NULL, + "updatedAt" timestamptz NOT NULL, + CONSTRAINT "eageEyeContents_pkey" PRIMARY KEY ("id") +); + +ALTER TABLE public."eageEyeContents" ADD CONSTRAINT "eageEyeContents_tenantId_fkey" FOREIGN KEY ("tenantId") REFERENCES public.tenants(id) ON DELETE NO ACTION ON UPDATE NO ACTION; + +CREATE TYPE "eagleEyeActions_action_type" AS ENUM ('thumbs-up', 'thumbs-down', 'bookmark'); + +CREATE TABLE public."eageEyeActions" ( + "id" uuid NOT NULL, + "platform" text NOT NULL, + "action" public."eagleEyeActions_actions_type" NOT NULL, + "timestamp" timestamptz NOT NULL, + "url" text NOT NULL, + "contentId" uuid NOT NULL, + "tenantId" uuid NOT NULL, + "actionBy" uuid NOT NULL, + "createdAt" timestamptz NOT NULL, + "updatedAt" timestamptz NOT NULL, + CONSTRAINT "eageEyeActions_pkey" PRIMARY KEY ("id") +); + +ALTER TABLE public."eageEyeActions" ADD CONSTRAINT "eageEyeActions_tenantId_fkey" FOREIGN KEY ("tenantId") REFERENCES public.tenants(id) ON DELETE NO ACTION ON UPDATE NO ACTION; +ALTER TABLE public."eageEyeActions" ADD CONSTRAINT "eageEyeActions_actionBy_fkey" FOREIGN KEY ("actionBy") REFERENCES public.users(id) ON DELETE NO ACTION ON UPDATE NO ACTION; +ALTER TABLE public."eageEyeActions" ADD CONSTRAINT "eageEyeActions_contentId_fkey" FOREIGN KEY ("contentId") REFERENCES public."eageEyeContents"(id) ON DELETE NO ACTION ON UPDATE NO ACTION; diff --git a/backend/src/database/models/eagleEyeAction.ts b/backend/src/database/models/eagleEyeAction.ts new file mode 100644 index 0000000000..305cef39e9 --- /dev/null +++ b/backend/src/database/models/eagleEyeAction.ts @@ -0,0 +1,62 @@ +import { DataTypes } from 'sequelize' + +const eagleEyeActionModel = { + id: { + type: DataTypes.UUID, + defaultValue: DataTypes.UUIDV4, + primaryKey: true, + }, + platform: { + type: DataTypes.TEXT, + allowNull: false, + validate: { + notEmpty: true, + }, + }, + action: { + type: DataTypes.TEXT, + validate: { + isIn: [['thumbs-up', 'thumbs-down', 'bookmark']], + }, + defaultValue: null, + }, + timestamp: { + type: DataTypes.DATE, + allowNull: false, + }, + url: { + type: DataTypes.TEXT, + allowNull: false, + validate: { + notEmpty: true, + }, + } +} + +export default (sequelize) => { + const eagleEyeAction = sequelize.define('eagleEyeAction', eagleEyeActionModel, { + timestamps: true, + paranoid: false, + }) + + eagleEyeAction.associate = (models) => { + models.eagleEyeContent.belongsTo(models.tenant, { + as: 'tenant', + foreignKey: { + allowNull: false, + }, + }) + + models.eagleEyeAction.belongsTo(models.user, { + as: 'actionBy', + }) + + models.eagleEyeAction.belongsTo(models.eagleEyeContent, { + as: 'content', + }) + } + + return eagleEyeAction +} + +export { eagleEyeActionModel } diff --git a/backend/src/database/models/eagleEyeContent.ts b/backend/src/database/models/eagleEyeContent.ts index bfee48a92e..d827d7972d 100644 --- a/backend/src/database/models/eagleEyeContent.ts +++ b/backend/src/database/models/eagleEyeContent.ts @@ -6,26 +6,31 @@ const eagleEyeContentModel = { defaultValue: DataTypes.UUIDV4, primaryKey: true, }, - sourceId: { + post: { + type: DataTypes.JSONB, + allowNull: false + }, + action: { type: DataTypes.TEXT, - allowNull: false, validate: { - notEmpty: true, + isIn: [['thumbs-up', 'thumbs-down', 'bookmark']], }, + defaultValue: null, + }, + timestamp: { + type: DataTypes.DATE, + allowNull: false, }, - vectorId: { + url: { type: DataTypes.TEXT, allowNull: false, validate: { notEmpty: true, }, }, - status: { - type: DataTypes.STRING(255), - validate: { - isIn: [['engaged', 'rejected']], - }, - defaultValue: null, + post: { + type: DataTypes.JSONB, + allowNull: false }, title: { type: DataTypes.TEXT, @@ -51,10 +56,7 @@ const eagleEyeContentModel = { text: { type: DataTypes.TEXT, }, - timestamp: { - type: DataTypes.DATE, - allowNull: false, - }, + platform: { type: DataTypes.TEXT, allowNull: false, @@ -75,7 +77,6 @@ const eagleEyeContentModel = { }, userAttributes: { type: DataTypes.JSONB, - default: {}, }, postAttributes: { type: DataTypes.JSONB, @@ -132,12 +133,9 @@ export default (sequelize) => { }) models.eagleEyeContent.belongsTo(models.user, { - as: 'createdBy', + as: 'actionBy', }) - models.eagleEyeContent.belongsTo(models.user, { - as: 'updatedBy', - }) } return eagleEyeContent From ffc51d3dcb3eb59a0e300e58d40435e14b8cc9ca Mon Sep 17 00:00:00 2001 From: anil Date: Thu, 2 Feb 2023 10:33:10 +0100 Subject: [PATCH 17/36] schema updates --- .../V1675259471__eagleEyeActions.sql | 8 +- .../src/database/models/eagleEyeContent.ts | 108 +----------------- 2 files changed, 5 insertions(+), 111 deletions(-) diff --git a/backend/src/database/migrations/V1675259471__eagleEyeActions.sql b/backend/src/database/migrations/V1675259471__eagleEyeActions.sql index 6ca96a3afb..96303250a4 100644 --- a/backend/src/database/migrations/V1675259471__eagleEyeActions.sql +++ b/backend/src/database/migrations/V1675259471__eagleEyeActions.sql @@ -11,22 +11,22 @@ CREATE TABLE public."eageEyeContents" ( ALTER TABLE public."eageEyeContents" ADD CONSTRAINT "eageEyeContents_tenantId_fkey" FOREIGN KEY ("tenantId") REFERENCES public.tenants(id) ON DELETE NO ACTION ON UPDATE NO ACTION; -CREATE TYPE "eagleEyeActions_action_type" AS ENUM ('thumbs-up', 'thumbs-down', 'bookmark'); +CREATE TYPE public."eagleEyeActions_action_type" AS ENUM ('thumbs-up', 'thumbs-down', 'bookmark'); CREATE TABLE public."eageEyeActions" ( "id" uuid NOT NULL, "platform" text NOT NULL, - "action" public."eagleEyeActions_actions_type" NOT NULL, + "action" public."eagleEyeActions_action_type" NOT NULL, "timestamp" timestamptz NOT NULL, "url" text NOT NULL, "contentId" uuid NOT NULL, "tenantId" uuid NOT NULL, - "actionBy" uuid NOT NULL, + "actionById" uuid NOT NULL, "createdAt" timestamptz NOT NULL, "updatedAt" timestamptz NOT NULL, CONSTRAINT "eageEyeActions_pkey" PRIMARY KEY ("id") ); ALTER TABLE public."eageEyeActions" ADD CONSTRAINT "eageEyeActions_tenantId_fkey" FOREIGN KEY ("tenantId") REFERENCES public.tenants(id) ON DELETE NO ACTION ON UPDATE NO ACTION; -ALTER TABLE public."eageEyeActions" ADD CONSTRAINT "eageEyeActions_actionBy_fkey" FOREIGN KEY ("actionBy") REFERENCES public.users(id) ON DELETE NO ACTION ON UPDATE NO ACTION; +ALTER TABLE public."eageEyeActions" ADD CONSTRAINT "eageEyeActions_actionBy_fkey" FOREIGN KEY ("actionById") REFERENCES public.users(id) ON DELETE NO ACTION ON UPDATE NO ACTION; ALTER TABLE public."eageEyeActions" ADD CONSTRAINT "eageEyeActions_contentId_fkey" FOREIGN KEY ("contentId") REFERENCES public."eageEyeContents"(id) ON DELETE NO ACTION ON UPDATE NO ACTION; diff --git a/backend/src/database/models/eagleEyeContent.ts b/backend/src/database/models/eagleEyeContent.ts index d827d7972d..f7cfae3e59 100644 --- a/backend/src/database/models/eagleEyeContent.ts +++ b/backend/src/database/models/eagleEyeContent.ts @@ -6,120 +6,14 @@ const eagleEyeContentModel = { defaultValue: DataTypes.UUIDV4, primaryKey: true, }, - post: { + content: { type: DataTypes.JSONB, allowNull: false }, - action: { - type: DataTypes.TEXT, - validate: { - isIn: [['thumbs-up', 'thumbs-down', 'bookmark']], - }, - defaultValue: null, - }, - timestamp: { - type: DataTypes.DATE, - allowNull: false, - }, - url: { - type: DataTypes.TEXT, - allowNull: false, - validate: { - notEmpty: true, - }, - }, - post: { - type: DataTypes.JSONB, - allowNull: false - }, - title: { - type: DataTypes.TEXT, - allowNull: false, - validate: { - notEmpty: true, - }, - }, - username: { - type: DataTypes.TEXT, - allowNull: false, - validate: { - notEmpty: true, - }, - }, - url: { - type: DataTypes.TEXT, - allowNull: false, - validate: { - notEmpty: true, - }, - }, - text: { - type: DataTypes.TEXT, - }, - - platform: { - type: DataTypes.TEXT, - allowNull: false, - validate: { - notEmpty: true, - }, - }, - keywords: { - type: DataTypes.ARRAY(DataTypes.TEXT), - default: [], - }, - exactKeywords: { - type: DataTypes.ARRAY(DataTypes.TEXT), - default: [], - }, - similarityScore: { - type: DataTypes.FLOAT, - }, - userAttributes: { - type: DataTypes.JSONB, - }, - postAttributes: { - type: DataTypes.JSONB, - default: {}, - }, - importHash: { - type: DataTypes.STRING(255), - allowNull: true, - validate: { - len: [0, 255], - }, - }, } export default (sequelize) => { const eagleEyeContent = sequelize.define('eagleEyeContent', eagleEyeContentModel, { - indexes: [ - { - unique: true, - fields: ['importHash', 'tenantId'], - where: { - deletedAt: null, - }, - }, - { - fields: ['platform', 'tenantId', 'timestamp'], - where: { - deletedAt: null, - }, - }, - { - fields: ['status', 'tenantId', 'timestamp'], - where: { - deletedAt: null, - }, - }, - { - fields: ['tenantId', 'timestamp'], - where: { - deletedAt: null, - }, - }, - ], timestamps: true, paranoid: true, }) From 6aa581b579cddf4eed9909c6efca62727059f5f7 Mon Sep 17 00:00:00 2001 From: anilb Date: Thu, 2 Feb 2023 12:37:08 +0100 Subject: [PATCH 18/36] schema updates, actions repo start --- .../V1675259471__eagleEyeActions.sql | 4 ++-- backend/src/database/models/eagleEyeAction.ts | 14 -------------- backend/src/database/models/eagleEyeContent.ts | 18 ++++++++++++++---- .../repositories/eagleEyeActionRepository.ts | 7 +++++++ 4 files changed, 23 insertions(+), 20 deletions(-) create mode 100644 backend/src/database/repositories/eagleEyeActionRepository.ts diff --git a/backend/src/database/migrations/V1675259471__eagleEyeActions.sql b/backend/src/database/migrations/V1675259471__eagleEyeActions.sql index 96303250a4..042f8d6cf2 100644 --- a/backend/src/database/migrations/V1675259471__eagleEyeActions.sql +++ b/backend/src/database/migrations/V1675259471__eagleEyeActions.sql @@ -2,6 +2,8 @@ DROP TABLE IF EXISTS "eagleEyeContents"; CREATE TABLE public."eageEyeContents" ( "id" uuid NOT NULL, + "platform" text NOT NULL, + "url" text NOT NULL, "content" jsonb NOT NULL, "tenantId" uuid NOT NULL, "createdAt" timestamptz NOT NULL, @@ -15,10 +17,8 @@ CREATE TYPE public."eagleEyeActions_action_type" AS ENUM ('thumbs-up', 'thumbs-d CREATE TABLE public."eageEyeActions" ( "id" uuid NOT NULL, - "platform" text NOT NULL, "action" public."eagleEyeActions_action_type" NOT NULL, "timestamp" timestamptz NOT NULL, - "url" text NOT NULL, "contentId" uuid NOT NULL, "tenantId" uuid NOT NULL, "actionById" uuid NOT NULL, diff --git a/backend/src/database/models/eagleEyeAction.ts b/backend/src/database/models/eagleEyeAction.ts index 305cef39e9..b47d8126b0 100644 --- a/backend/src/database/models/eagleEyeAction.ts +++ b/backend/src/database/models/eagleEyeAction.ts @@ -6,13 +6,6 @@ const eagleEyeActionModel = { defaultValue: DataTypes.UUIDV4, primaryKey: true, }, - platform: { - type: DataTypes.TEXT, - allowNull: false, - validate: { - notEmpty: true, - }, - }, action: { type: DataTypes.TEXT, validate: { @@ -24,13 +17,6 @@ const eagleEyeActionModel = { type: DataTypes.DATE, allowNull: false, }, - url: { - type: DataTypes.TEXT, - allowNull: false, - validate: { - notEmpty: true, - }, - } } export default (sequelize) => { diff --git a/backend/src/database/models/eagleEyeContent.ts b/backend/src/database/models/eagleEyeContent.ts index f7cfae3e59..8e7e7ab702 100644 --- a/backend/src/database/models/eagleEyeContent.ts +++ b/backend/src/database/models/eagleEyeContent.ts @@ -6,10 +6,24 @@ const eagleEyeContentModel = { defaultValue: DataTypes.UUIDV4, primaryKey: true, }, + platform: { + type: DataTypes.TEXT, + allowNull: false, + validate: { + notEmpty: true, + }, + }, content: { type: DataTypes.JSONB, allowNull: false }, + url: { + type: DataTypes.TEXT, + allowNull: false, + validate: { + notEmpty: true, + }, + } } export default (sequelize) => { @@ -26,10 +40,6 @@ export default (sequelize) => { }, }) - models.eagleEyeContent.belongsTo(models.user, { - as: 'actionBy', - }) - } return eagleEyeContent diff --git a/backend/src/database/repositories/eagleEyeActionRepository.ts b/backend/src/database/repositories/eagleEyeActionRepository.ts new file mode 100644 index 0000000000..8fb48cce8b --- /dev/null +++ b/backend/src/database/repositories/eagleEyeActionRepository.ts @@ -0,0 +1,7 @@ +import { IRepositoryOptions } from './IRepositoryOptions' + +export default class EagleEyeActionRepository { + + + +} \ No newline at end of file From 94f4ddd0a4598f05e9763e9267d01c717389af76 Mon Sep 17 00:00:00 2001 From: anilb Date: Thu, 2 Feb 2023 18:18:50 +0100 Subject: [PATCH 19/36] repo layer progress --- .../V1675259471__eagleEyeActions.sql | 2 +- .../src/database/models/eagleEyeContent.ts | 2 +- .../repositories/eagleEyeContentRepository.ts | 367 +++--------------- backend/src/types/eagleEyeTypes.ts | 24 ++ 4 files changed, 84 insertions(+), 311 deletions(-) create mode 100644 backend/src/types/eagleEyeTypes.ts diff --git a/backend/src/database/migrations/V1675259471__eagleEyeActions.sql b/backend/src/database/migrations/V1675259471__eagleEyeActions.sql index 042f8d6cf2..6ec06fe385 100644 --- a/backend/src/database/migrations/V1675259471__eagleEyeActions.sql +++ b/backend/src/database/migrations/V1675259471__eagleEyeActions.sql @@ -4,7 +4,7 @@ CREATE TABLE public."eageEyeContents" ( "id" uuid NOT NULL, "platform" text NOT NULL, "url" text NOT NULL, - "content" jsonb NOT NULL, + "post" jsonb NOT NULL, "tenantId" uuid NOT NULL, "createdAt" timestamptz NOT NULL, "updatedAt" timestamptz NOT NULL, diff --git a/backend/src/database/models/eagleEyeContent.ts b/backend/src/database/models/eagleEyeContent.ts index 8e7e7ab702..8b11be5333 100644 --- a/backend/src/database/models/eagleEyeContent.ts +++ b/backend/src/database/models/eagleEyeContent.ts @@ -13,7 +13,7 @@ const eagleEyeContentModel = { notEmpty: true, }, }, - content: { + post: { type: DataTypes.JSONB, allowNull: false }, diff --git a/backend/src/database/repositories/eagleEyeContentRepository.ts b/backend/src/database/repositories/eagleEyeContentRepository.ts index 56de3e60f0..ca7aff5eef 100644 --- a/backend/src/database/repositories/eagleEyeContentRepository.ts +++ b/backend/src/database/repositories/eagleEyeContentRepository.ts @@ -5,8 +5,7 @@ import Error404 from '../../errors/Error404' import Error400 from '../../errors/Error400' import AuditLogRepository from './auditLogRepository' import { IRepositoryOptions } from './IRepositoryOptions' -import QueryParser from './filters/queryParser' -import { QueryOutput } from './filters/queryTypes' +import { EagleEyeContentData } from '../../types/eagleEyeTypes' export default class EagleEyeContentRepository { /** @@ -15,300 +14,49 @@ export default class EagleEyeContentRepository { * @param options Repository options. * @returns Created EagleEyeContent record. */ - static async upsert(data, options: IRepositoryOptions) { - const currentUser = SequelizeRepository.getCurrentUser(options) - - const tenant = SequelizeRepository.getCurrentTenant(options) - - const transaction = SequelizeRepository.getTransaction(options) + static async upsert(data:EagleEyeContentData, options: IRepositoryOptions): Promise { - if (data.status && ![null, 'rejected', 'engaged'].includes(data.status)) { - throw new Error400('en', 'errors.invalidEagleEyeStatus.message') + if(!data.url){ + throw new Error(`Can't upsert without url`) } - const existing = await options.database.eagleEyeContent.findOne({ - where: { - tenantId: tenant.id, - sourceId: data.sourceId, - }, - }) - - // If the content is already shown, we don't need to add it again - if (existing) { - // If the content comes from a different kewword, we also add it - if (!lodash.isEqual(data.keywords.sort(), existing.keywords.sort())) { - const keywords = lodash.uniq([...existing.keywords, ...data.keywords]) - let exactKeywords = null - if (data.exactKeywords && !existing.exactKeywords) { - exactKeywords = data.exactKeywords - } else if (!data.exactKeywords && existing.exactKeywords) { - exactKeywords = existing.exactKeywords - } - if (data.exactKeywords && existing.exactKeywords) { - exactKeywords = lodash.uniq([...existing.exactKeywords, ...data.exactKeywords]) - } - const similarityScore = data.similarityScore - return existing.update( - { - keywords, - similarityScore, - exactKeywords, - }, - { - transaction, - }, - ) - } - return existing - } + // find by url + const existing = await EagleEyeContentRepository.findByUrl(data.url, options) - if (typeof data.keywords === 'string') { - data.keywords = [data.keywords] - } + let record - if (typeof data.exactKeywords === 'string') { - data.exactKeywords = [data.exactKeywords] + if (existing){ + record = await EagleEyeContentRepository.update(existing.id, data, options) } - - if (typeof data.timestamp === 'number') { - data.timestamp = moment.unix(data.timestamp).toDate() + /* + else{ + record = options.database.eagleEyeContent.create( + { + ...lodash.pick(data, [ + 'platform', + 'post', + 'url', + ]), + memberId: data.member || null, + parentId: data.parent || null, + sourceParentId: data.sourceParentId || null, + conversationId: data.conversationId || null, + tenantId: tenant.id, + createdById: currentUser.id, + updatedById: currentUser.id, + }, + { + transaction, + }, + ) } + */ - const record = await options.database.eagleEyeContent.create( - { - ...lodash.pick(data, [ - 'sourceId', - 'vectorId', - 'status', - 'postAttributes', - 'title', - 'username', - 'url', - 'text', - 'timestamp', - 'userAttributes', - 'platform', - 'keywords', - 'exactKeywords', - 'similarityScore', - 'importHash', - ]), - - tenantId: tenant.id, - createdById: currentUser.id, - updatedById: currentUser.id, - }, - { - transaction, - }, - ) - - await AuditLogRepository.log( - { - entityName: 'eagleEyeContent', - entityId: record.id, - action: AuditLogRepository.CREATE, - values: data, - }, - options, - ) + return this.findById(record.id, options) } - /** - * EagleEyeContent find all records matching given criteria. - * @returns Records found. - */ - static async findAndCountAll( - { filter = {} as any, advancedFilter = null as any, limit = 0, offset = 0, orderBy = '' }, - options: IRepositoryOptions, - ) { - // If the advanced filter is empty, we construct it from the query parameter filter - if (!advancedFilter) { - advancedFilter = { and: [] } - - if (filter.id) { - advancedFilter.and.push({ - id: filter.id, - }) - } - - if (filter.sourceId) { - advancedFilter.and.push({ - sourceId: filter.sourceId, - }) - } - - if (filter.vectorId) { - advancedFilter.and.push({ - vectorId: filter.vectorId, - }) - } - - if (filter.status) { - if (filter.status === 'NULL') { - advancedFilter.and.push({ - status: 'NULL', - }) - } else if (filter.status === 'NOT_NULL') { - advancedFilter.and.push({ - status: { - not: null, - }, - }) - } else { - advancedFilter.and.push({ - status: { - textContains: filter.status, - }, - }) - } - } - - if (filter.timestampRange) { - const [start, end] = filter.timestampRange - - if (start !== undefined && start !== null && start !== '') { - advancedFilter.and.push({ - timestamp: { - gte: start, - }, - }) - } - - if (end !== undefined && end !== null && end !== '') { - advancedFilter.and.push({ - timestamp: { - lte: end, - }, - }) - } - } - - if (filter.platforms) { - advancedFilter.and.push({ - platform: { - or: filter.platforms.split(','), - }, - }) - } - - if (filter.nDays) { - advancedFilter.and.push({ - timestamp: { - gte: moment().subtract(filter.nDays, 'days').toDate(), - }, - }) - } - - if (filter.title) { - advancedFilter.and.push({ - title: { - textContains: filter.title, - }, - }) - } - - if (filter.text) { - advancedFilter.and.push({ - text: { - textContains: filter.text, - }, - }) - } - - if (filter.url) { - advancedFilter.and.push({ - url: { - textContains: filter.url, - }, - }) - } - - if (filter.username) { - advancedFilter.and.push({ - username: { - textContains: filter.username, - }, - }) - } - - if (filter.keywords) { - // Overlap will take a post where any keyword matches any of the filter keywords - advancedFilter.and.push({ - keywords: { - overlap: filter.keywords.split(','), - }, - }) - } - - if (filter.similarityScoreRange) { - const [start, end] = filter.similarityScoreRange - - if (start !== undefined && start !== null && start !== '') { - advancedFilter.and.push({ - similarityScore: { - gte: start, - }, - }) - } - - if (end !== undefined && end !== null && end !== '') { - advancedFilter.and.push({ - similarityScore: { - lte: end, - }, - }) - } - } - - if (filter.createdAtRange) { - const [start, end] = filter.createdAtRange - - if (start !== undefined && start !== null && start !== '') { - advancedFilter.and.push({ - createdAt: { - gte: start, - }, - }) - } - - if (end !== undefined && end !== null && end !== '') { - advancedFilter.and.push({ - createdAt: { - lte: end, - }, - }) - } - } - } - - const parser = new QueryParser({}, options) - - const parsed: QueryOutput = parser.parse({ - filter: advancedFilter, - orderBy: orderBy || ['createdAt_DESC'], - limit, - offset, - }) - - let { - rows, - count, // eslint-disable-line prefer-const - } = await options.database.eagleEyeContent.findAndCountAll({ - ...(parsed.where ? { where: parsed.where } : {}), - ...(parsed.having ? { having: parsed.having } : {}), - order: parsed.order, - limit: parsed.limit, - offset: parsed.offset, - transaction: SequelizeRepository.getTransaction(options), - }) - - rows = await this._populateRelationsForRows(rows) - - return { rows, count, limit: parsed.limit, offset: parsed.offset } - } static async update(id, data, options: IRepositoryOptions) { const currentUser = SequelizeRepository.getCurrentUser(options) @@ -329,26 +77,13 @@ export default class EagleEyeContentRepository { throw new Error404() } - if (data.status && ![null, 'rejected', 'engaged'].includes(data.status)) { - throw new Error400('en', 'errors.invalidEagleEyeStatus.message') - } record = await record.update( { ...lodash.pick(data, [ - 'sourceId', - 'vectorId', - 'status', - 'title', - 'username', - 'url', - 'text', - 'postAttributes', - 'timestamp', 'platform', - 'userAttributes', - 'importHash', - // Missing keywords on purpose + 'post', + 'url', ]), updatedById: currentUser.id, }, @@ -357,19 +92,10 @@ export default class EagleEyeContentRepository { }, ) - await AuditLogRepository.log( - { - entityName: 'eagleEyeContent', - entityId: record.id, - action: AuditLogRepository.UPDATE, - values: data, - }, - options, - ) return this.findById(record.id, options) } - static async findById(id, options: IRepositoryOptions) { + static async findById(id:string, options: IRepositoryOptions) { const transaction = SequelizeRepository.getTransaction(options) const include = [] @@ -392,6 +118,29 @@ export default class EagleEyeContentRepository { return this._populateRelations(record) } + static async findByUrl(url:string, options: IRepositoryOptions) { + const transaction = SequelizeRepository.getTransaction(options) + + const include = [] + + const currentTenant = SequelizeRepository.getCurrentTenant(options) + + const record = await options.database.eagleEyeContent.findOne({ + where: { + url, + tenantId: currentTenant.id, + }, + include, + transaction, + }) + + if (!record) { + return null + } + + return this._populateRelations(record) + } + static async _populateRelationsForRows(rows) { if (!rows) { return rows diff --git a/backend/src/types/eagleEyeTypes.ts b/backend/src/types/eagleEyeTypes.ts new file mode 100644 index 0000000000..0ffbaaf1f2 --- /dev/null +++ b/backend/src/types/eagleEyeTypes.ts @@ -0,0 +1,24 @@ +export enum EagleEyeActionType { + THUMBS_UP = 'thumbs-up', + THUMBS_DOWN = 'thumbs-down', + BOOKMARK = 'bookmark', +} + +export interface EagleEyeContentData { + id?: string + platform: string + post: any + url: string + tenantId: string + createdAt?: string + updatedAt?: string +} + +export interface EagleEyeActionData { + id?: string + action: EagleEyeActionType + timestamp: string + content: EagleEyeContentData + createdAt?: string + updatedAt?: string +} From bbe8c2031df6538c4d7a446d826755ee24fab655 Mon Sep 17 00:00:00 2001 From: anil Date: Sun, 5 Feb 2023 01:58:29 +0100 Subject: [PATCH 20/36] filtering with tests, endpoint routing remaining --- .../eagleEyeContent/eagleEyeActionCreate.ts | 0 .../eagleEyeContent/eagleEyeContentUpsert.ts | 0 backend/src/api/eagleEyeContent/index.ts | 6 + .../V1675259471__eagleEyeActions.sql | 21 +- backend/src/database/models/eagleEyeAction.ts | 4 +- .../src/database/models/eagleEyeContent.ts | 12 +- backend/src/database/models/index.ts | 1 + .../eagleEyeActionRepository.test.ts | 61 +++ .../eagleEyeContentRepository.test.ts | 371 ++++++++---------- .../repositories/eagleEyeActionRepository.ts | 122 +++++- .../repositories/eagleEyeContentRepository.ts | 204 +++++++--- .../src/database/utils/sequelizeTestUtils.ts | 1 + .../__tests__/eagleEyeContentService.test.ts | 149 ++----- backend/src/services/eagleEyeActionService.ts | 79 ++++ .../src/services/eagleEyeContentService.ts | 178 ++------- backend/src/types/common.ts | 7 + backend/src/types/eagleEyeTypes.ts | 25 +- 17 files changed, 698 insertions(+), 543 deletions(-) create mode 100644 backend/src/api/eagleEyeContent/eagleEyeActionCreate.ts create mode 100644 backend/src/api/eagleEyeContent/eagleEyeContentUpsert.ts create mode 100644 backend/src/database/repositories/__tests__/eagleEyeActionRepository.test.ts create mode 100644 backend/src/services/eagleEyeActionService.ts diff --git a/backend/src/api/eagleEyeContent/eagleEyeActionCreate.ts b/backend/src/api/eagleEyeContent/eagleEyeActionCreate.ts new file mode 100644 index 0000000000..e69de29bb2 diff --git a/backend/src/api/eagleEyeContent/eagleEyeContentUpsert.ts b/backend/src/api/eagleEyeContent/eagleEyeContentUpsert.ts new file mode 100644 index 0000000000..e69de29bb2 diff --git a/backend/src/api/eagleEyeContent/index.ts b/backend/src/api/eagleEyeContent/index.ts index 33928e9148..5cd3220e19 100644 --- a/backend/src/api/eagleEyeContent/index.ts +++ b/backend/src/api/eagleEyeContent/index.ts @@ -18,4 +18,10 @@ export default (app) => { `/tenant/:tenantId/eagleEyeContent/:id`, safeWrap(require('./eagleEyeContentFind').default), ) + + app.post( + `/tenant/:tenantId/eagleEyeContent/:contentId/action`, + safeWrap(require('./eagleEyeActionCreate').default), + ) + } diff --git a/backend/src/database/migrations/V1675259471__eagleEyeActions.sql b/backend/src/database/migrations/V1675259471__eagleEyeActions.sql index 6ec06fe385..9183281bc1 100644 --- a/backend/src/database/migrations/V1675259471__eagleEyeActions.sql +++ b/backend/src/database/migrations/V1675259471__eagleEyeActions.sql @@ -1,32 +1,33 @@ DROP TABLE IF EXISTS "eagleEyeContents"; -CREATE TABLE public."eageEyeContents" ( +CREATE TABLE public."eagleEyeContents" ( "id" uuid NOT NULL, "platform" text NOT NULL, "url" text NOT NULL, "post" jsonb NOT NULL, "tenantId" uuid NOT NULL, + "postedAt" timestamptz NOT NULL, "createdAt" timestamptz NOT NULL, "updatedAt" timestamptz NOT NULL, - CONSTRAINT "eageEyeContents_pkey" PRIMARY KEY ("id") + CONSTRAINT "eagleEyeContents_pkey" PRIMARY KEY ("id") ); -ALTER TABLE public."eageEyeContents" ADD CONSTRAINT "eageEyeContents_tenantId_fkey" FOREIGN KEY ("tenantId") REFERENCES public.tenants(id) ON DELETE NO ACTION ON UPDATE NO ACTION; +ALTER TABLE public."eagleEyeContents" ADD CONSTRAINT "eagleEyeContents_tenantId_fkey" FOREIGN KEY ("tenantId") REFERENCES public.tenants(id) ON DELETE NO ACTION ON UPDATE NO ACTION; -CREATE TYPE public."eagleEyeActions_action_type" AS ENUM ('thumbs-up', 'thumbs-down', 'bookmark'); +CREATE TYPE public."eagleEyeActionTypes_type" AS ENUM ('thumbs-up', 'thumbs-down', 'bookmark'); -CREATE TABLE public."eageEyeActions" ( +CREATE TABLE public."eagleEyeActions" ( "id" uuid NOT NULL, - "action" public."eagleEyeActions_action_type" NOT NULL, + "type" public."eagleEyeActionTypes_type" NOT NULL, "timestamp" timestamptz NOT NULL, "contentId" uuid NOT NULL, "tenantId" uuid NOT NULL, "actionById" uuid NOT NULL, "createdAt" timestamptz NOT NULL, "updatedAt" timestamptz NOT NULL, - CONSTRAINT "eageEyeActions_pkey" PRIMARY KEY ("id") + CONSTRAINT "eagleEyeActions_pkey" PRIMARY KEY ("id") ); -ALTER TABLE public."eageEyeActions" ADD CONSTRAINT "eageEyeActions_tenantId_fkey" FOREIGN KEY ("tenantId") REFERENCES public.tenants(id) ON DELETE NO ACTION ON UPDATE NO ACTION; -ALTER TABLE public."eageEyeActions" ADD CONSTRAINT "eageEyeActions_actionBy_fkey" FOREIGN KEY ("actionById") REFERENCES public.users(id) ON DELETE NO ACTION ON UPDATE NO ACTION; -ALTER TABLE public."eageEyeActions" ADD CONSTRAINT "eageEyeActions_contentId_fkey" FOREIGN KEY ("contentId") REFERENCES public."eageEyeContents"(id) ON DELETE NO ACTION ON UPDATE NO ACTION; +ALTER TABLE public."eagleEyeActions" ADD CONSTRAINT "eagleEyeActions_tenantId_fkey" FOREIGN KEY ("tenantId") REFERENCES public.tenants(id) ON DELETE NO ACTION ON UPDATE NO ACTION; +ALTER TABLE public."eagleEyeActions" ADD CONSTRAINT "eagleEyeActions_actionBy_fkey" FOREIGN KEY ("actionById") REFERENCES public.users(id) ON DELETE NO ACTION ON UPDATE NO ACTION; +ALTER TABLE public."eagleEyeActions" ADD CONSTRAINT "eagleEyeActions_contentId_fkey" FOREIGN KEY ("contentId") REFERENCES public."eagleEyeContents"(id) ON DELETE CASCADE ON UPDATE NO ACTION; diff --git a/backend/src/database/models/eagleEyeAction.ts b/backend/src/database/models/eagleEyeAction.ts index b47d8126b0..14eaab4260 100644 --- a/backend/src/database/models/eagleEyeAction.ts +++ b/backend/src/database/models/eagleEyeAction.ts @@ -6,7 +6,7 @@ const eagleEyeActionModel = { defaultValue: DataTypes.UUIDV4, primaryKey: true, }, - action: { + type: { type: DataTypes.TEXT, validate: { isIn: [['thumbs-up', 'thumbs-down', 'bookmark']], @@ -26,7 +26,7 @@ export default (sequelize) => { }) eagleEyeAction.associate = (models) => { - models.eagleEyeContent.belongsTo(models.tenant, { + models.eagleEyeAction.belongsTo(models.tenant, { as: 'tenant', foreignKey: { allowNull: false, diff --git a/backend/src/database/models/eagleEyeContent.ts b/backend/src/database/models/eagleEyeContent.ts index 8b11be5333..198f28e754 100644 --- a/backend/src/database/models/eagleEyeContent.ts +++ b/backend/src/database/models/eagleEyeContent.ts @@ -23,13 +23,17 @@ const eagleEyeContentModel = { validate: { notEmpty: true, }, - } + }, + postedAt: { + type: DataTypes.DATE, + allowNull: false, + }, } export default (sequelize) => { const eagleEyeContent = sequelize.define('eagleEyeContent', eagleEyeContentModel, { timestamps: true, - paranoid: true, + paranoid: false, }) eagleEyeContent.associate = (models) => { @@ -39,6 +43,10 @@ export default (sequelize) => { allowNull: false, }, }) + models.eagleEyeContent.hasMany(models.eagleEyeAction, { + as: 'actions', + foreignKey: 'contentId', + }) } diff --git a/backend/src/database/models/index.ts b/backend/src/database/models/index.ts index 9a1dcc704c..4e0b3bcc09 100644 --- a/backend/src/database/models/index.ts +++ b/backend/src/database/models/index.ts @@ -102,6 +102,7 @@ function models() { require('./conversation').default, require('./conversationSettings').default, require('./eagleEyeContent').default, + require('./eagleEyeAction').default, require('./automation').default, require('./automationExecution').default, require('./organization').default, diff --git a/backend/src/database/repositories/__tests__/eagleEyeActionRepository.test.ts b/backend/src/database/repositories/__tests__/eagleEyeActionRepository.test.ts new file mode 100644 index 0000000000..f3e035cc20 --- /dev/null +++ b/backend/src/database/repositories/__tests__/eagleEyeActionRepository.test.ts @@ -0,0 +1,61 @@ +import EagleEyeContentRepository from '../eagleEyeContentRepository' +import SequelizeTestUtils from '../../utils/sequelizeTestUtils' +import { EagleEyeAction, EagleEyeActionType, EagleEyeContent } from '../../../types/eagleEyeTypes' +import EagleEyeActionRepository from '../eagleEyeActionRepository' + +const db = null + +describe('eagleEyeActionRepository tests', () => { + beforeEach(async () => { + await SequelizeTestUtils.wipeDatabase(db) + }) + + afterAll((done) => { + // Closing the DB connection allows Jest to exit successfully. + SequelizeTestUtils.closeConnection(db) + done() + }) + + describe('createActionForContent method', () => { + it('Should create a an action for a content succesfully', async () => { + const mockIRepositoryOptions = await SequelizeTestUtils.getTestIRepositoryOptions(db) + + const content = { + platform: 'reddit', + url: 'https://some-post-url', + post: { + title: 'post title', + body: 'post body', + }, + postedAt: '2020-05-27T15:13:30Z', + tenantId: mockIRepositoryOptions.currentTenant.id, + } as EagleEyeContent + + const contentCreated = await EagleEyeContentRepository.create(content, mockIRepositoryOptions) + + + const action: EagleEyeAction= { + type: EagleEyeActionType.BOOKMARK, + timestamp: '2022-07-27T19:13:30Z', + } + + const actionCreated = await EagleEyeActionRepository.createActionForContent(action, contentCreated.id, mockIRepositoryOptions) + + actionCreated.createdAt = (actionCreated.createdAt as Date).toISOString().split('T')[0] + actionCreated.updatedAt = (actionCreated.updatedAt as Date).toISOString().split('T')[0] + + const expectedAction = { + id: actionCreated.id, + ...action, + timestamp: new Date(actionCreated.timestamp), + contentId: contentCreated.id, + actionById: mockIRepositoryOptions.currentUser.id, + tenantId: mockIRepositoryOptions.currentTenant.id, + createdAt: SequelizeTestUtils.getNowWithoutTime(), + updatedAt: SequelizeTestUtils.getNowWithoutTime(), + } + expect(expectedAction).toStrictEqual(actionCreated) + }) + + }) +}) diff --git a/backend/src/database/repositories/__tests__/eagleEyeContentRepository.test.ts b/backend/src/database/repositories/__tests__/eagleEyeContentRepository.test.ts index bffec870ca..eef92a3679 100644 --- a/backend/src/database/repositories/__tests__/eagleEyeContentRepository.test.ts +++ b/backend/src/database/repositories/__tests__/eagleEyeContentRepository.test.ts @@ -1,85 +1,11 @@ -import lodash from 'lodash' -import moment from 'moment' import EagleEyeContentRepository from '../eagleEyeContentRepository' import SequelizeTestUtils from '../../utils/sequelizeTestUtils' -import Error404 from '../../../errors/Error404' -import Error400 from '../../../errors/Error400' -import EagleEyeContentService from '../../../services/eagleEyeContentService' -import { PlatformType } from '../../../types/integrationEnums' +import { EagleEyeActionType, EagleEyeContent } from '../../../types/eagleEyeTypes' +import EagleEyeActionRepository from '../eagleEyeActionRepository' + -const db = null -const toCreate = { - sourceId: 'sourceId', - vectorId: '123', - status: null, - platform: 'hacker_news', - title: 'title', - userAttributes: { [PlatformType.GITHUB]: 'hey', [PlatformType.TWITTER]: 'ho' }, - text: 'text', - postAttributes: { - score: 10, - }, - url: 'url', - exactKeywords: null, - timestamp: new Date(), - username: 'username', - keywords: ['keyword1', 'keyword2'], - similarityScore: 0.9, -} - -const toCreateMinimal = { - sourceId: 'sourceIdMinimal', - vectorId: '456', - platform: 'hacker_news', - url: 'url', - title: 'title minimal', - timestamp: new Date(), - username: 'username', - keywords: 'keyword', -} - -const forFiltering = [ - toCreate, - toCreateMinimal, - { - sourceId: 'devto123', - vectorId: '123123', - status: 'engaged', - url: 'devto url', - username: 'devtousername1', - platform: 'devto', - timestamp: moment().toDate(), - title: 'title devto 1', - }, - { - sourceId: 'devto456', - vectorId: '123456', - url: 'url devto 2', - username: 'devtousername2', - status: 'rejected', - platform: 'devto', - timestamp: moment().subtract(1, 'week').toDate(), - title: 'title devto 2', - keywords: ['keyword1', 'keyword2'], - score: 40, - }, - { - sourceId: 'devto789', - vectorId: '123456', - url: 'url devto 3', - username: 'devtousername3', - status: null, - platform: 'devto', - timestamp: moment().subtract(1, 'week').toDate(), - keywords: ['keyword3', 'keyword2'], - title: 'title devto 3', - }, -] - -async function addAll(options) { - await Promise.all(forFiltering.map((item) => EagleEyeContentRepository.upsert(item, options))) -} +const db = null describe('eagleEyeContentRepository tests', () => { beforeEach(async () => { @@ -92,29 +18,40 @@ describe('eagleEyeContentRepository tests', () => { done() }) - describe('upserts method', () => { - it('Should create a complete content succesfully', async () => { + describe('create method', () => { + it('Should create a content succesfully', async () => { const mockIRepositoryOptions = await SequelizeTestUtils.getTestIRepositoryOptions(db) - const created = await EagleEyeContentRepository.upsert(toCreate, mockIRepositoryOptions) + const content = { + platform: 'reddit', + url: 'https://some-post-url', + post: { + title: 'post title', + body: 'post body', + }, + postedAt: '2020-05-27T15:13:30Z', + tenantId: mockIRepositoryOptions.currentTenant.id, + } as EagleEyeContent - created.createdAt = created.createdAt.toISOString().split('T')[0] - created.updatedAt = created.updatedAt.toISOString().split('T')[0] + const created = await EagleEyeContentRepository.create(content, mockIRepositoryOptions) + + created.createdAt = (created.createdAt as Date).toISOString().split('T')[0] + created.updatedAt = (created.updatedAt as Date).toISOString().split('T')[0] const expectedCreated = { id: created.id, - ...toCreate, - importHash: null, + ...content, + postedAt: new Date(content.postedAt), + actions: [], createdAt: SequelizeTestUtils.getNowWithoutTime(), updatedAt: SequelizeTestUtils.getNowWithoutTime(), - deletedAt: null, tenantId: mockIRepositoryOptions.currentTenant.id, - createdById: mockIRepositoryOptions.currentUser.id, - updatedById: mockIRepositoryOptions.currentUser.id, } expect(created).toStrictEqual(expectedCreated) }) + /* + it('Should create a content with unix timestamp', async () => { const mockIRepositoryOptions = await SequelizeTestUtils.getTestIRepositoryOptions(db) @@ -157,130 +94,9 @@ describe('eagleEyeContentRepository tests', () => { expect(created).toStrictEqual(expectedCreated) }) - it('Should not add it the record already exists', async () => { - const mockIRepositoryOptions = await SequelizeTestUtils.getTestIRepositoryOptions(db) - - await EagleEyeContentRepository.upsert(toCreate, mockIRepositoryOptions) - - await EagleEyeContentRepository.upsert(toCreate, mockIRepositoryOptions) - - const count = await mockIRepositoryOptions.database.eagleEyeContent.count() - expect(count).toBe(1) - }) - - it('Should update keywords and similarity score if the item already exists', async () => { - const mockIRepositoryOptions = await SequelizeTestUtils.getTestIRepositoryOptions(db) - - await EagleEyeContentRepository.upsert(toCreate, mockIRepositoryOptions) - - const toCreateNewKeywords = { ...toCreate } - toCreateNewKeywords.keywords = ['1', '2', 'keyword1'] - toCreateNewKeywords.similarityScore = 0.95 - - const allKeywords = ['1', '2', 'keyword1', 'keyword2'] - - const created = await EagleEyeContentRepository.upsert( - toCreateNewKeywords, - mockIRepositoryOptions, - ) - - const count = await mockIRepositoryOptions.database.eagleEyeContent.count() - expect(count).toBe(1) - expect(lodash.isEqual(created.keywords.sort(), allKeywords.sort())) - expect(created.similarityScore).toBe(0.95) - }) - - it('Should create a minimal content succesfully', async () => { - const mockIRepositoryOptions = await SequelizeTestUtils.getTestIRepositoryOptions(db) - - const created = await EagleEyeContentRepository.upsert( - toCreateMinimal, - mockIRepositoryOptions, - ) - - created.createdAt = created.createdAt.toISOString().split('T')[0] - created.updatedAt = created.updatedAt.toISOString().split('T')[0] - - const expectedCreated = { - id: created.id, - ...toCreateMinimal, - text: null, - status: null, - userAttributes: null, - postAttributes: null, - similarityScore: null, - exactKeywords: null, - importHash: null, - createdAt: SequelizeTestUtils.getNowWithoutTime(), - updatedAt: SequelizeTestUtils.getNowWithoutTime(), - deletedAt: null, - tenantId: mockIRepositoryOptions.currentTenant.id, - createdById: mockIRepositoryOptions.currentUser.id, - updatedById: mockIRepositoryOptions.currentUser.id, - } - expect(created).toStrictEqual(expectedCreated) - expect(created.status).toBe(null) - }) - - it('Should create with rejected status', async () => { - const mockIRepositoryOptions = await SequelizeTestUtils.getTestIRepositoryOptions(db) - const newStatus = { ...toCreate } - newStatus.status = 'rejected' - - const created = await EagleEyeContentRepository.upsert(newStatus, mockIRepositoryOptions) - - created.createdAt = created.createdAt.toISOString().split('T')[0] - created.updatedAt = created.updatedAt.toISOString().split('T')[0] - - const expectedCreated = { - id: created.id, - ...newStatus, - importHash: null, - createdAt: SequelizeTestUtils.getNowWithoutTime(), - updatedAt: SequelizeTestUtils.getNowWithoutTime(), - deletedAt: null, - tenantId: mockIRepositoryOptions.currentTenant.id, - createdById: mockIRepositoryOptions.currentUser.id, - updatedById: mockIRepositoryOptions.currentUser.id, - } - expect(created).toStrictEqual(expectedCreated) - expect(created.status).toBe('rejected') - }) - - it('Should create with engaged status', async () => { - const mockIRepositoryOptions = await SequelizeTestUtils.getTestIRepositoryOptions(db) - const newStatus = { ...toCreate } - newStatus.status = 'engaged' - - const created = await EagleEyeContentRepository.upsert(newStatus, mockIRepositoryOptions) - - created.createdAt = created.createdAt.toISOString().split('T')[0] - created.updatedAt = created.updatedAt.toISOString().split('T')[0] - - const expectedCreated = { - id: created.id, - ...newStatus, - importHash: null, - createdAt: SequelizeTestUtils.getNowWithoutTime(), - updatedAt: SequelizeTestUtils.getNowWithoutTime(), - deletedAt: null, - tenantId: mockIRepositoryOptions.currentTenant.id, - createdById: mockIRepositoryOptions.currentUser.id, - updatedById: mockIRepositoryOptions.currentUser.id, - } - expect(created).toStrictEqual(expectedCreated) - expect(created.status).toBe('engaged') - }) - - it('Should throw an error for an invalid status', async () => { - const mockIRepositoryOptions = await SequelizeTestUtils.getTestIRepositoryOptions(db) - const newStatus = { ...toCreate } - newStatus.status = 'smth else' + - await expect(() => - EagleEyeContentRepository.upsert(newStatus, mockIRepositoryOptions), - ).rejects.toThrowError(new Error400('en', 'errors.invalidEagleEyeStatus.message')) - }) + }) describe('find by id method', () => { @@ -601,4 +417,137 @@ describe('eagleEyeContentRepository tests', () => { expect(updated.keywords).toStrictEqual(created.keywords) }) }) + */ + }) + + describe('findAndCountAll method', () => { + it('Should find eagle eye contant, various cases', async () => { + // create random tenant with one user + const mockIRepositoryOptions = await SequelizeTestUtils.getTestIRepositoryOptions(db) + + // create additional users for same tenant to test out actionBy filtering + const randomUser = await SequelizeTestUtils.getRandomUser() + + console.log("random user: ") + console.log(randomUser) + + const user2 = await mockIRepositoryOptions.database.user.create(randomUser) + + await mockIRepositoryOptions.database.tenantUser.create({ + roles: ['admin'], + status: 'active', + tenantId: mockIRepositoryOptions.currentTenant.id, + userId: user2.id, + }) + + // create few content + // one without any actions + await EagleEyeContentRepository.create( + { + platform: 'reddit', + url: 'https://some-reddit-url', + post: { + title: 'post title', + body: 'post body', + }, + postedAt: '2020-05-27T15:13:30Z', + tenantId: mockIRepositoryOptions.currentTenant.id, + }, + mockIRepositoryOptions, + ) + + + // one with a bookmark action + let c2 = await EagleEyeContentRepository.create( + { + platform: 'hackernews', + url: 'https://some-hackernews-url', + post: { + title: 'post title', + body: 'post body', + }, + postedAt: '2022-06-27T19:14:44Z', + tenantId: mockIRepositoryOptions.currentTenant.id, + }, + mockIRepositoryOptions, + ) + + // add bookmark action + await EagleEyeActionRepository.createActionForContent( + { + type: EagleEyeActionType.BOOKMARK, + timestamp: '2022-07-27T19:13:30Z', + }, + c2.id, + mockIRepositoryOptions, + ) + + c2 = await EagleEyeContentRepository.findById(c2.id, mockIRepositoryOptions) + + // another content with a thumbs-up(user1) and a bookmark(user2) action + let c3 = await EagleEyeContentRepository.create( + { + platform: 'devto', + url: 'https://some-devto-url', + post: { + title: 'post title', + body: 'post body', + }, + postedAt: '2022-06-27T19:14:44Z', + tenantId: mockIRepositoryOptions.currentTenant.id, + }, + mockIRepositoryOptions, + ) + + // add the thumbs up action + await EagleEyeActionRepository.createActionForContent( + { + type: EagleEyeActionType.THUMBS_UP, + timestamp: '2022-09-30T23:11:10Z', + }, + c3.id, + mockIRepositoryOptions, + ) + + // also add bookmark from user2 + await EagleEyeActionRepository.createActionForContent( + { + type: EagleEyeActionType.BOOKMARK, + timestamp: '2022-09-30T23:11:10Z', + }, + c3.id, + {...mockIRepositoryOptions, currentUser: user2}, + ) + + c3 = await EagleEyeContentRepository.findById(c3.id, mockIRepositoryOptions) + + + // filter by action type + let res = await EagleEyeContentRepository.findAndCountAll({ + advancedFilter: { + action: { + type: EagleEyeActionType.BOOKMARK + + } + } + }, mockIRepositoryOptions) + + + expect(res.count).toBe(2) + expect(res.rows.sort((a, b) => (a.createdAt > b.createdAt ? 1 : -1))).toStrictEqual([c2,c3]) + + // filter by actionBy + res = await EagleEyeContentRepository.findAndCountAll({ + advancedFilter: { + action: { + actionById: user2.id + } + } + }, mockIRepositoryOptions) + + expect(res.count).toBe(1) + expect(res.rows).toStrictEqual([c3]) + + }) + }) }) diff --git a/backend/src/database/repositories/eagleEyeActionRepository.ts b/backend/src/database/repositories/eagleEyeActionRepository.ts index 8fb48cce8b..cc7932b257 100644 --- a/backend/src/database/repositories/eagleEyeActionRepository.ts +++ b/backend/src/database/repositories/eagleEyeActionRepository.ts @@ -1,7 +1,125 @@ +import lodash from 'lodash' +import Error404 from '../../errors/Error404' +import { EagleEyeAction, EagleEyeActionType } from '../../types/eagleEyeTypes' import { IRepositoryOptions } from './IRepositoryOptions' +import SequelizeRepository from './sequelizeRepository' export default class EagleEyeActionRepository { + static async createActionForContent( + data: EagleEyeAction, + contentId: string, + options: IRepositoryOptions, + ): Promise { + const currentUser = SequelizeRepository.getCurrentUser(options) - + const transaction = SequelizeRepository.getTransaction(options) -} \ No newline at end of file + const currentTenant = SequelizeRepository.getCurrentTenant(options) + + const record = await options.database.eagleEyeAction.create( + { + ...lodash.pick(data, ['type', 'timestamp']), + actionById: currentUser.id, + contentId, + tenantId: currentTenant.id, + }, + { + transaction, + }, + ) + + return this.findById(record.id, options) + } + + static async removeActionFromContent( + action: EagleEyeActionType, + contentId: string, + options: IRepositoryOptions, + ) { + const currentUser = SequelizeRepository.getCurrentUser(options) + + const currentTenant = SequelizeRepository.getCurrentTenant(options) + + const transaction = SequelizeRepository.getTransaction(options) + + const record = await options.database.eagleEyeAction.findOne({ + where: { + contentId, + action, + actionById: currentUser.id, + tenantId: currentTenant.id, + }, + transaction, + }) + + if (record) { + await record.destroy({ + transaction, + force: true, + }) + } + } + + static async destroy(id: string, options: IRepositoryOptions): Promise { + const transaction = SequelizeRepository.getTransaction(options) + + const currentTenant = SequelizeRepository.getCurrentTenant(options) + + const record = await options.database.eagleEyeAction.findOne({ + where: { + id, + tenantId: currentTenant.id, + }, + transaction, + }) + + if (record) { + await record.destroy({ + transaction, + force: true, + }) + } + } + + static async findById(id: string, options: IRepositoryOptions) { + const transaction = SequelizeRepository.getTransaction(options) + + const include = [] + + const currentTenant = SequelizeRepository.getCurrentTenant(options) + + const record = await options.database.eagleEyeAction.findOne({ + where: { + id, + tenantId: currentTenant.id, + }, + include, + transaction, + }) + + if (!record) { + throw new Error404() + } + + return this._populateRelations(record) + } + + static async create(data: EagleEyeAction, options: IRepositoryOptions): Promise { + const currentTenant = SequelizeRepository.getCurrentTenant(options) + + const record = options.database.eagleEyeContent.create({ + ...lodash.pick(data, ['type', 'timestamp']), + tenantId: currentTenant.id, + }) + + return this.findById(record.id, options) + } + + static async _populateRelations(record) { + if (!record) { + return record + } + + return record.get({ plain: true }) + } +} diff --git a/backend/src/database/repositories/eagleEyeContentRepository.ts b/backend/src/database/repositories/eagleEyeContentRepository.ts index ca7aff5eef..73dcf99ee5 100644 --- a/backend/src/database/repositories/eagleEyeContentRepository.ts +++ b/backend/src/database/repositories/eagleEyeContentRepository.ts @@ -1,60 +1,39 @@ -import moment from 'moment' import lodash from 'lodash' +import { Op } from 'sequelize' import SequelizeRepository from './sequelizeRepository' import Error404 from '../../errors/Error404' -import Error400 from '../../errors/Error400' -import AuditLogRepository from './auditLogRepository' import { IRepositoryOptions } from './IRepositoryOptions' -import { EagleEyeContentData } from '../../types/eagleEyeTypes' +import { EagleEyeContent } from '../../types/eagleEyeTypes' +import QueryParser from './filters/queryParser' +import { QueryOutput } from './filters/queryTypes' +import SequelizeFilterUtils from '../utils/sequelizeFilterUtils' +import EagleEyeActionRepository from './eagleEyeActionRepository' export default class EagleEyeContentRepository { - /** - * Create an eagle eye shown content record. - * @param data Data to a new EagleEyeContent record. - * @param options Repository options. - * @returns Created EagleEyeContent record. - */ - static async upsert(data:EagleEyeContentData, options: IRepositoryOptions): Promise { - - if(!data.url){ - throw new Error(`Can't upsert without url`) - } - // find by url - const existing = await EagleEyeContentRepository.findByUrl(data.url, options) + static async create(data:EagleEyeContent, options:IRepositoryOptions): Promise{ + const currentTenant = SequelizeRepository.getCurrentTenant(options) - let record + const record = await options.database.eagleEyeContent.create( + { + ...lodash.pick(data, [ + 'platform', + 'post', + 'url', + 'postedAt' + ]), + tenantId: currentTenant.id, + }, + ) - if (existing){ - record = await EagleEyeContentRepository.update(existing.id, data, options) - } - /* - else{ - record = options.database.eagleEyeContent.create( - { - ...lodash.pick(data, [ - 'platform', - 'post', - 'url', - ]), - memberId: data.member || null, - parentId: data.parent || null, - sourceParentId: data.sourceParentId || null, - conversationId: data.conversationId || null, - tenantId: tenant.id, - createdById: currentUser.id, - updatedById: currentUser.id, - }, - { - transaction, - }, - ) + if (data.actions){ + for (const action of data.actions){ + await EagleEyeActionRepository.createActionForContent(action, record.id, options) + } } - */ - - return this.findById(record.id, options) + } @@ -83,6 +62,7 @@ export default class EagleEyeContentRepository { ...lodash.pick(data, [ 'platform', 'post', + 'postedAt', 'url', ]), updatedById: currentUser.id, @@ -98,7 +78,14 @@ export default class EagleEyeContentRepository { static async findById(id:string, options: IRepositoryOptions) { const transaction = SequelizeRepository.getTransaction(options) - const include = [] + const include = [{ + model: options.database.eagleEyeAction, + as: 'actions', + // attributes: [], + // through: { + // attributes: [], + // }, + }] const currentTenant = SequelizeRepository.getCurrentTenant(options) @@ -118,6 +105,131 @@ export default class EagleEyeContentRepository { return this._populateRelations(record) } + static async destroy(id: string, options: IRepositoryOptions): Promise { + const transaction = SequelizeRepository.getTransaction(options) + + const currentTenant = SequelizeRepository.getCurrentTenant(options) + + const record = await options.database.eagleEyeContent.findOne({ + where: { + id, + tenantId: currentTenant.id, + }, + transaction, + }) + + if (record) { + await record.destroy({ + transaction, + force: true, + }) + } + } + + static async findAndCountAll( + { + advancedFilter = null as any, + limit = 0, + offset = 0, + orderBy = '', + }, + options: IRepositoryOptions, + ) { + + const actionsSequelizeInclude = { + model: options.database.eagleEyeAction, + as: 'actions', + // required:false, + where: {}, + // subQuery:true + + } + + // let wh = {} + + if (advancedFilter && advancedFilter.action) { + + const actionQueryParser = new QueryParser( + { + // nestedFields: { + // type: `$actions.type$` + // } + }, + options, + ) + + const parsedActionQuery: QueryOutput = actionQueryParser.parse({ + filter: advancedFilter.action, + orderBy: 'timestamp_DESC', + }) + + actionsSequelizeInclude.where = parsedActionQuery.where ?? {} + delete advancedFilter.action + } + + const include = [ + actionsSequelizeInclude, + ] + + const contentParser = new QueryParser( + {}, + options, + ) + + + const parsed: QueryOutput = contentParser.parse({ + filter: advancedFilter, + orderBy: orderBy || ['postedAt_DESC'], + limit, + offset, + }) + + + + console.log("SENDING SHIEEEEEEEEEET") + console.log(parsed) + console.log(include) + + // const wtf = parsed.where ? { where: {...parsed.where, ...wh} } : {} + console.log("wtf") + // console.log(wtf) + let { + rows, + count, // eslint-disable-line prefer-const + } = await options.database.eagleEyeContent.findAndCountAll({ + include, + ...(parsed.where ? { where: parsed.where } : {}), + order: parsed.order, + limit: parsed.limit, + offset: parsed.offset, + transaction: SequelizeRepository.getTransaction(options), + subQuery: false + }) + + // If we have an actions filter, we should query again to eager + // load the all actions on a content because previous query will + // omit actions that don't match the given action filter + if (Object.keys(actionsSequelizeInclude.where).length !== 0) + { + rows = (await options.database.eagleEyeContent.findAndCountAll({ + include: [{...actionsSequelizeInclude, where: {}}], + where: { id: { [Op.in]: rows.map( (i) => i.id) } }, + order: parsed.order, + limit: parsed.limit, + offset: parsed.offset, + transaction: SequelizeRepository.getTransaction(options), + subQuery: false + + })).rows + } + + + rows = await this._populateRelationsForRows(rows) + + return { rows, count, limit: parsed.limit, offset: parsed.offset } + + } + static async findByUrl(url:string, options: IRepositoryOptions) { const transaction = SequelizeRepository.getTransaction(options) diff --git a/backend/src/database/utils/sequelizeTestUtils.ts b/backend/src/database/utils/sequelizeTestUtils.ts index 7b103981f4..83e813d9a2 100644 --- a/backend/src/database/utils/sequelizeTestUtils.ts +++ b/backend/src/database/utils/sequelizeTestUtils.ts @@ -34,6 +34,7 @@ export default class SequelizeTestUtils { files, microservices, "eagleEyeContents", + "eagleEyeActions", "auditLogs", "memberEnrichmentCache" cascade; diff --git a/backend/src/services/__tests__/eagleEyeContentService.test.ts b/backend/src/services/__tests__/eagleEyeContentService.test.ts index 46b7d51a8b..f67db5118e 100644 --- a/backend/src/services/__tests__/eagleEyeContentService.test.ts +++ b/backend/src/services/__tests__/eagleEyeContentService.test.ts @@ -1,44 +1,9 @@ -import moment from 'moment' import SequelizeTestUtils from '../../database/utils/sequelizeTestUtils' -import { PlatformType } from '../../types/integrationEnums' +import { EagleEyeActionType, EagleEyeContent } from '../../types/eagleEyeTypes' import EagleEyeContentService from '../eagleEyeContentService' const db = null -const toUpsert = { - keywords: ['keyword'], - similarityScore: 1, - userAttributes: { - [PlatformType.GITHUB]: 'github', - }, - username: 'username', - timestamp: moment().unix(), - url: 'url', - title: 'title', - text: 'text', - platform: 'platform', - sourceId: 'sourceId', - vectorId: '1234', - status: null, -} - -const toUpsert2 = { - keywords: ['keyword'], - similarityScore: 1, - userAttributes: { - [PlatformType.GITHUB]: 'github', - }, - username: 'username', - timestamp: moment().subtract(1, 'days').unix(), - url: 'url', - title: 'title', - text: 'text', - platform: 'platform', - sourceId: 'sourceId2', - vectorId: '12345', - status: null, -} - describe('EagleEyeContentService tests', () => { beforeEach(async () => { await SequelizeTestUtils.wipeDatabase(db) @@ -49,91 +14,57 @@ describe('EagleEyeContentService tests', () => { await SequelizeTestUtils.closeConnection(db) }) - describe('bulk upsert method', () => { - it('Should upsert a single record', async () => { + describe('upsert method', () => { + it('Should create or update a single content using URL field', async () => { const mockIRepositoryOptions = await SequelizeTestUtils.getTestIRepositoryOptions(db) - const service = new EagleEyeContentService(mockIRepositoryOptions) - await service.bulkUpsert([toUpsert]) - const result = await service.findAndCountAll({}) - expect(result.count).toBe(1) - }) - it('Should upsert a single record', async () => { - const mockIRepositoryOptions = await SequelizeTestUtils.getTestIRepositoryOptions(db) - const service = new EagleEyeContentService(mockIRepositoryOptions) - await service.bulkUpsert([toUpsert]) - await service.bulkUpsert([toUpsert2]) - const result = await service.findAndCountAll({}) - expect(result.count).toBe(2) - }) - }) - - describe('findAndCount all method', () => { - it('Should find records', async () => { - const mockIRepositoryOptions = await SequelizeTestUtils.getTestIRepositoryOptions(db) - const service = new EagleEyeContentService(mockIRepositoryOptions) - await service.bulkUpsert([toUpsert]) - await service.bulkUpsert([toUpsert2]) - const result = await service.findAndCountAll({}) - - expect(result.count).toBe(2) - expect(result.rows[1].vectorId).toBe(toUpsert.vectorId) - expect(result.rows[0].vectorId).toBe(toUpsert2.vectorId) - }) + const content: EagleEyeContent = { + platform: 'reddit', + url: 'https://some-post-url', + post: { + title: 'post title', + body: 'post body', + }, + postedAt: '2020-05-27T15:13:30Z', + tenantId: mockIRepositoryOptions.currentTenant.id, + actions: [{ + type: EagleEyeActionType.BOOKMARK, + timestamp: '2022-06-27T14:13:30Z', + }] + } - it('Should work when no records', async () => { - const mockIRepositoryOptions = await SequelizeTestUtils.getTestIRepositoryOptions(db) const service = new EagleEyeContentService(mockIRepositoryOptions) - const result = await service.findAndCountAll({}) - expect(result.count).toBe(0) - }) - }) + const c1 = await service.upsert(content) - describe('findNotInbox method', () => { - it('4 records: 2 have status null, one is too old. Return 1', async () => { - const mockIRepositoryOptions = await SequelizeTestUtils.getTestIRepositoryOptions(db) - const service = new EagleEyeContentService(mockIRepositoryOptions) + let contents = await service.query({}) - const nInbox1 = { - keywords: ['keyword'], - similarityScore: 1, - userAttributes: { - [PlatformType.GITHUB]: 'github', - }, - username: 'username', - timestamp: moment().unix(), - url: 'url', - title: 'title', - text: 'text', - platform: 'platform', - sourceId: 'p-321', - vectorId: '321', - status: 'rejected', - } + expect(contents.count).toBe(1) + expect(contents.rows).toStrictEqual([c1]) - const nInbox2 = { - keywords: ['keyword'], - similarityScore: 1, - userAttributes: { - [PlatformType.GITHUB]: 'github', + // upsert previous url with some new fields + const contentWithSameUrl: EagleEyeContent = { + platform: 'reddit', + url: 'https://some-post-url', + post: { + title: 'a brand new post title', + body: 'better post body', }, - username: 'username', - timestamp: moment().subtract(31, 'days').unix(), - url: 'url', - title: 'title', - text: 'text', - platform: 'platform', - sourceId: 'p-4321', - vectorId: '4321', - status: 'engaged', + postedAt: '2020-05-27T15:13:30Z', + tenantId: mockIRepositoryOptions.currentTenant.id, } - await service.bulkUpsert([toUpsert, nInbox1, nInbox2, toUpsert2]) + const c1Upserted = await service.upsert(contentWithSameUrl) - const result = await service.findNotInbox() + contents = await service.query({}) + expect(contents.count).toBe(1) + expect(contents.rows).toStrictEqual([c1Upserted]) + expect(c1Upserted.id).toEqual(c1.id) + expect(contents.rows[0].post).toStrictEqual(contentWithSameUrl.post) - expect(result.length).toBe(1) - expect(result[0]).toBe(nInbox1.vectorId) }) + + }) + + }) diff --git a/backend/src/services/eagleEyeActionService.ts b/backend/src/services/eagleEyeActionService.ts new file mode 100644 index 0000000000..7711915543 --- /dev/null +++ b/backend/src/services/eagleEyeActionService.ts @@ -0,0 +1,79 @@ +import EagleEyeActionRepository from '../database/repositories/eagleEyeActionRepository' +import EagleEyeContentRepository from '../database/repositories/eagleEyeContentRepository' +import Error404 from '../errors/Error404' +import { EagleEyeAction, EagleEyeActionType } from '../types/eagleEyeTypes' +import { IServiceOptions } from './IServiceOptions' +import { LoggingBase } from './loggingBase' + +export default class EagleEyeActionService extends LoggingBase { + options: IServiceOptions + + constructor(options) { + super(options) + this.options = options + } + + async create(data: EagleEyeAction, contentId: string): Promise { + // find content + const content = await EagleEyeContentRepository.findById(contentId, this.options) + + if (!content) { + throw new Error404('Content not found..') + } + + const existingUserActions: EagleEyeAction[] = content.actions + .filter((a) => a.actionById === this.options.currentUser.id) + + const existingUserActionTypes = existingUserActions.map((a) => a.type) + + // check if already bookmarked - if yes ignore the new action and return existing + if ( + data.type === EagleEyeActionType.BOOKMARK && + existingUserActionTypes.includes(EagleEyeActionType.BOOKMARK) + ) { + return existingUserActions.find( (a) => a.type === EagleEyeActionType.BOOKMARK) + } + + // thumbs up and down should be mutually exclusive + if ( + data.type === EagleEyeActionType.THUMBS_DOWN && + existingUserActionTypes.includes(EagleEyeActionType.THUMBS_UP) + ) { + await EagleEyeActionRepository.removeActionFromContent( + EagleEyeActionType.THUMBS_UP, + contentId, + this.options, + ) + } else if ( + data.type === EagleEyeActionType.THUMBS_UP && + existingUserActionTypes.includes(EagleEyeActionType.THUMBS_DOWN) + ) { + await EagleEyeActionRepository.removeActionFromContent( + EagleEyeActionType.THUMBS_DOWN, + contentId, + this.options, + ) + } + + // add new action + return EagleEyeActionRepository.createActionForContent(data, contentId, this.options) + } + + + async destroy(id: string){ + + const action = await EagleEyeActionRepository.findById(id, this.options) + + const contentId = action.contentId + + await EagleEyeActionRepository.destroy(id, this.options) + + // find content + const content = await EagleEyeContentRepository.findById(contentId, this.options) + + // if content no longer has any actions attached to it, also delete the content + if (content.actions.length === 0){ + await EagleEyeContentRepository.destroy(contentId, this.options) + } + } +} diff --git a/backend/src/services/eagleEyeContentService.ts b/backend/src/services/eagleEyeContentService.ts index e42de0de9a..69bfd7f132 100644 --- a/backend/src/services/eagleEyeContentService.ts +++ b/backend/src/services/eagleEyeContentService.ts @@ -1,30 +1,13 @@ -import moment from 'moment' -import request from 'superagent' -import { API_CONFIG } from '../config' -import SequelizeRepository from '../database/repositories/sequelizeRepository' import { IServiceOptions } from './IServiceOptions' import EagleEyeContentRepository from '../database/repositories/eagleEyeContentRepository' -import Error400 from '../errors/Error400' -import track from '../segment/track' import { LoggingBase } from './loggingBase' +import { EagleEyeContent, EagleEyeAction } from '../types/eagleEyeTypes' +import { PageData, QueryData } from '../types/common' -interface EagleEyeSearchPoint { - vectorId: string - sourceId: string - title: string - text?: string - url: string - timestamp: number - username: string - similarityScore: number - userAttributes: { - [platform: string]: string - } - keywords: string[] +export interface EagleEyeContentUpsertData extends EagleEyeAction { + content: EagleEyeContent } -type EagleEyeSearchOutput = EagleEyeSearchPoint[] - export default class EagleEyeContentService extends LoggingBase { options: IServiceOptions @@ -33,51 +16,36 @@ export default class EagleEyeContentService extends LoggingBase { this.options = options } - async upsert(data) { - const transaction = await SequelizeRepository.createTransaction(this.options) - - try { - const record = await EagleEyeContentRepository.upsert(data, { - ...this.options, - transaction, - }) - - await SequelizeRepository.commitTransaction(transaction) + /** + * Create an eagle eye shown content record. + * @param data Data to a new EagleEyeContent record. + * @param options Repository options. + * @returns Created EagleEyeContent record. + */ + async upsert(data: EagleEyeContent): Promise { + if (!data.url) { + throw new Error(`Can't upsert without url`) + } - return record - } catch (error) { - await SequelizeRepository.rollbackTransaction(transaction) + // find by url + const existing = await EagleEyeContentRepository.findByUrl(data.url, this.options) - SequelizeRepository.handleUniqueFieldError(error, this.options.language, 'EagleEyeContent') + let record - throw error + if (existing) { + record = await EagleEyeContentRepository.update(existing.id, data, this.options) + } else { + record = await EagleEyeContentRepository.create(data, this.options) } - } - async findNotInbox() { - const shown = ( - await EagleEyeContentRepository.findAndCountAll( - { - filter: { - timestampRange: [ - moment().subtract(30, 'days').toDate(), - moment().add(1, 'hour').toDate(), - ], - status: 'NOT_NULL', - }, - }, - this.options, - ) - ).rows - // Slicing results such that lambda payload will not be too big - return shown.map((record) => record.vectorId).slice(0, 20000) + return record } - async findAndCountAll(args) { - return EagleEyeContentRepository.findAndCountAll(args, this.options) + async findById(id: string): Promise { + return EagleEyeContentRepository.findById(id, this.options) } - async query(data) { + async query(data: QueryData): Promise> { const advancedFilter = data.filter const orderBy = data.orderBy const limit = data.limit @@ -88,99 +56,9 @@ export default class EagleEyeContentService extends LoggingBase { ) } - async bulkUpsert(data: EagleEyeSearchOutput) { - for (const point of data) { - await this.upsert(point) - } - } - + /** TODO async search(args) { - const { keywords, nDays, exactKeywords } = args - // We do not want what we have already accepted or rejected - const filters = await this.findNotInbox() - if (API_CONFIG.premiumApiUrl) { - const response = await request - .post(`${API_CONFIG.premiumApiUrl}/search`) - .send({ queries: keywords, nDays, filters, exactKeywords }) - const fromEagleEye: EagleEyeSearchOutput = JSON.parse(response.text) - await this.bulkUpsert(fromEagleEye) - return fromEagleEye - } - return [] as EagleEyeSearchOutput - } - - async keywordMatch(args) { - const { keywords, nDays, platform } = args - - if (API_CONFIG.premiumApiUrl) { - const response = await request - .post(`${API_CONFIG.premiumApiUrl}/keyword-match`) - .send({ exactKeywords: keywords, nDays, platform }) - try { - return JSON.parse(response.text) - } catch (error) { - this.log.error({ error: response.error }, 'error while calling eagle eye server!') - throw new Error400('en', 'errors.eagleEyeSearchFailed.message') - } - } else { - return [] as EagleEyeSearchOutput - } - } - - async update(id, data) { - const transaction = await SequelizeRepository.createTransaction(this.options) - - try { - const recordBeforeUpdate = await EagleEyeContentRepository.findById(id, { ...this.options }) - const record = await EagleEyeContentRepository.update(id, data, { - ...this.options, - transaction, - }) - - // If we are updating status we want to track it - if (data.status !== recordBeforeUpdate.status) { - // If we are going from null to status, we are either accepting or rejecting - if (data.status && data.status !== null && data.status !== undefined) { - track( - `EagleEye ${data.status}`, - { - ...data, - platform: record.platform, - keywords: record.keywords, - title: record.title, - url: record.url, - }, - { ...this.options }, - ) - // Here we are bringing back a rejected post to the Inbox - } else if (recordBeforeUpdate.status === 'rejected' && data.status === null) { - track( - `EagleEye post from rejected to Inbox`, - { - ...data, - platform: record.platform, - keywords: record.keywords, - title: record.title, - url: record.url, - }, - { ...this.options }, - ) - } - } - - await SequelizeRepository.commitTransaction(transaction) - - return record - } catch (error) { - await SequelizeRepository.rollbackTransaction(transaction) - - SequelizeRepository.handleUniqueFieldError(error, this.options.language, 'EagleEyeContent') - - throw error - } - } - - async findById(id) { - return EagleEyeContentRepository.findById(id, this.options) + } + */ } diff --git a/backend/src/types/common.ts b/backend/src/types/common.ts index 1eb58e2dc9..d7b4b7a253 100644 --- a/backend/src/types/common.ts +++ b/backend/src/types/common.ts @@ -5,6 +5,13 @@ export interface PageData { offset: number } +export interface QueryData { + filter?: any + orderBy?: string + limit?: number + offset?: number +} + export interface SearchCriteria { limit?: number offset?: number diff --git a/backend/src/types/eagleEyeTypes.ts b/backend/src/types/eagleEyeTypes.ts index 0ffbaaf1f2..556ed1fd8c 100644 --- a/backend/src/types/eagleEyeTypes.ts +++ b/backend/src/types/eagleEyeTypes.ts @@ -4,21 +4,24 @@ export enum EagleEyeActionType { BOOKMARK = 'bookmark', } -export interface EagleEyeContentData { + +export interface EagleEyeAction { + id?: string + type: EagleEyeActionType + timestamp: Date | string + createdAt?: Date | string + updatedAt?: Date | string +} + +export interface EagleEyeContent { id?: string platform: string post: any url: string + actions?: EagleEyeAction[] tenantId: string - createdAt?: string - updatedAt?: string + postedAt: string + createdAt?: Date | string + updatedAt?: Date | string } -export interface EagleEyeActionData { - id?: string - action: EagleEyeActionType - timestamp: string - content: EagleEyeContentData - createdAt?: string - updatedAt?: string -} From 5462fd87879477437c53985a19595252171a866e Mon Sep 17 00:00:00 2001 From: anilb Date: Sun, 5 Feb 2023 20:00:31 +0100 Subject: [PATCH 21/36] comments --- .../repositories/eagleEyeContentRepository.ts | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/backend/src/database/repositories/eagleEyeContentRepository.ts b/backend/src/database/repositories/eagleEyeContentRepository.ts index 73dcf99ee5..66b87609f6 100644 --- a/backend/src/database/repositories/eagleEyeContentRepository.ts +++ b/backend/src/database/repositories/eagleEyeContentRepository.ts @@ -6,7 +6,6 @@ import { IRepositoryOptions } from './IRepositoryOptions' import { EagleEyeContent } from '../../types/eagleEyeTypes' import QueryParser from './filters/queryParser' import { QueryOutput } from './filters/queryTypes' -import SequelizeFilterUtils from '../utils/sequelizeFilterUtils' import EagleEyeActionRepository from './eagleEyeActionRepository' export default class EagleEyeContentRepository { @@ -81,10 +80,6 @@ export default class EagleEyeContentRepository { const include = [{ model: options.database.eagleEyeAction, as: 'actions', - // attributes: [], - // through: { - // attributes: [], - // }, }] const currentTenant = SequelizeRepository.getCurrentTenant(options) @@ -139,21 +134,15 @@ export default class EagleEyeContentRepository { const actionsSequelizeInclude = { model: options.database.eagleEyeAction, as: 'actions', - // required:false, where: {}, - // subQuery:true } - // let wh = {} if (advancedFilter && advancedFilter.action) { const actionQueryParser = new QueryParser( { - // nestedFields: { - // type: `$actions.type$` - // } }, options, ) @@ -185,14 +174,6 @@ export default class EagleEyeContentRepository { }) - - console.log("SENDING SHIEEEEEEEEEET") - console.log(parsed) - console.log(include) - - // const wtf = parsed.where ? { where: {...parsed.where, ...wh} } : {} - console.log("wtf") - // console.log(wtf) let { rows, count, // eslint-disable-line prefer-const From 661fb7ad44d442740dee08deede20d610cc37e41 Mon Sep 17 00:00:00 2001 From: anilb Date: Sun, 5 Feb 2023 20:33:37 +0100 Subject: [PATCH 22/36] eagleEye upsert routing --- .../src/api/eagleEyeContent/eagleEyeContentUpsert.ts | 11 +++++++++++ backend/src/security/permissions.ts | 5 +++++ 2 files changed, 16 insertions(+) diff --git a/backend/src/api/eagleEyeContent/eagleEyeContentUpsert.ts b/backend/src/api/eagleEyeContent/eagleEyeContentUpsert.ts index e69de29bb2..2acfcc5e2d 100644 --- a/backend/src/api/eagleEyeContent/eagleEyeContentUpsert.ts +++ b/backend/src/api/eagleEyeContent/eagleEyeContentUpsert.ts @@ -0,0 +1,11 @@ +import Permissions from '../../security/permissions' +import EagleEyeContentService from '../../services/eagleEyeContentService' +import PermissionChecker from '../../services/user/permissionChecker' + +export default async (req, res) => { + new PermissionChecker(req).validateHas(Permissions.values.eagleEyeContentCreate) + + const payload = await new EagleEyeContentService(req).upsert(req.body) + + await req.responseHandler.success(req, res, payload) +} diff --git a/backend/src/security/permissions.ts b/backend/src/security/permissions.ts index 3c75df4c2b..74482a3f03 100644 --- a/backend/src/security/permissions.ts +++ b/backend/src/security/permissions.ts @@ -422,6 +422,11 @@ class Permissions { allowedRoles: [roles.admin, roles.readonly], allowedPlans: [plans.essential, plans.growth], }, + eagleEyeContentCreate: { + id: 'eagleEyeContentCreate', + allowedRoles: [roles.admin], + allowedPlans: [plans.growth], + }, eagleEyeContentRead: { id: 'eagleEyeContentRead', allowedRoles: [roles.admin, roles.readonly], From ff078092be247de141fc5920c4f407863f903343 Mon Sep 17 00:00:00 2001 From: anilb Date: Sun, 5 Feb 2023 20:37:09 +0100 Subject: [PATCH 23/36] transactions for upsert --- .../src/services/eagleEyeContentService.ts | 38 +++++++++++++------ 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/backend/src/services/eagleEyeContentService.ts b/backend/src/services/eagleEyeContentService.ts index 69bfd7f132..e78e2eb93a 100644 --- a/backend/src/services/eagleEyeContentService.ts +++ b/backend/src/services/eagleEyeContentService.ts @@ -3,6 +3,7 @@ import EagleEyeContentRepository from '../database/repositories/eagleEyeContentR import { LoggingBase } from './loggingBase' import { EagleEyeContent, EagleEyeAction } from '../types/eagleEyeTypes' import { PageData, QueryData } from '../types/common' +import SequelizeRepository from '../database/repositories/sequelizeRepository' export interface EagleEyeContentUpsertData extends EagleEyeAction { content: EagleEyeContent @@ -23,22 +24,37 @@ export default class EagleEyeContentService extends LoggingBase { * @returns Created EagleEyeContent record. */ async upsert(data: EagleEyeContent): Promise { - if (!data.url) { - throw new Error(`Can't upsert without url`) - } - // find by url - const existing = await EagleEyeContentRepository.findByUrl(data.url, this.options) + const transaction = await SequelizeRepository.createTransaction(this.options) + + try { + + if (!data.url) { + throw new Error(`Can't upsert without url`) + } + + // find by url + const existing = await EagleEyeContentRepository.findByUrl(data.url, this.options) + + let record + + if (existing) { + record = await EagleEyeContentRepository.update(existing.id, data, this.options) + } else { + record = await EagleEyeContentRepository.create(data, this.options) + } + + await SequelizeRepository.commitTransaction(transaction) + + return record + } catch (error) { + await SequelizeRepository.rollbackTransaction(transaction) - let record + SequelizeRepository.handleUniqueFieldError(error, this.options.language, 'EagleEyeContent') - if (existing) { - record = await EagleEyeContentRepository.update(existing.id, data, this.options) - } else { - record = await EagleEyeContentRepository.create(data, this.options) + throw error } - return record } async findById(id: string): Promise { From 792932a8caf02e487d516265307341d5e4839376 Mon Sep 17 00:00:00 2001 From: anilb Date: Sun, 5 Feb 2023 20:46:15 +0100 Subject: [PATCH 24/36] error translations --- backend/src/i18n/en.ts | 10 ++--- .../src/services/eagleEyeContentService.ts | 39 +++++++------------ 2 files changed, 16 insertions(+), 33 deletions(-) diff --git a/backend/src/i18n/en.ts b/backend/src/i18n/en.ts index 0a41b33134..0c4a89b705 100644 --- a/backend/src/i18n/en.ts +++ b/backend/src/i18n/en.ts @@ -102,19 +102,15 @@ const en = { activityDup: { message: 'This activity has already been linked to this member', }, - invalidEagleEyeStatus: { - message: 'Possible statuses are: "shown", "rejected", "engaged"', - }, - eagleEyeSearchFailed: { - message: 'Search failed in EagleEye', - }, OrganizationNameRequired: { message: 'Organization Name is required', }, projectNotFound: { message: 'Project not found', }, - + eagleEye: { + urlRequiredWhenUpserting: 'URL field is mandatory when upserting eagleEyeContent' + }, integrations: { badEndpoint: 'Bad endpoint: {0}', }, diff --git a/backend/src/services/eagleEyeContentService.ts b/backend/src/services/eagleEyeContentService.ts index e78e2eb93a..fdbd133d55 100644 --- a/backend/src/services/eagleEyeContentService.ts +++ b/backend/src/services/eagleEyeContentService.ts @@ -3,7 +3,7 @@ import EagleEyeContentRepository from '../database/repositories/eagleEyeContentR import { LoggingBase } from './loggingBase' import { EagleEyeContent, EagleEyeAction } from '../types/eagleEyeTypes' import { PageData, QueryData } from '../types/common' -import SequelizeRepository from '../database/repositories/sequelizeRepository' +import Error400 from '../errors/Error400' export interface EagleEyeContentUpsertData extends EagleEyeAction { content: EagleEyeContent @@ -25,36 +25,23 @@ export default class EagleEyeContentService extends LoggingBase { */ async upsert(data: EagleEyeContent): Promise { - const transaction = await SequelizeRepository.createTransaction(this.options) - - try { - - if (!data.url) { - throw new Error(`Can't upsert without url`) - } - - // find by url - const existing = await EagleEyeContentRepository.findByUrl(data.url, this.options) - - let record - - if (existing) { - record = await EagleEyeContentRepository.update(existing.id, data, this.options) - } else { - record = await EagleEyeContentRepository.create(data, this.options) - } - - await SequelizeRepository.commitTransaction(transaction) + if (!data.url) { + throw new Error400(this.options.language, 'errors.eagleEye.urlRequiredWhenUpserting') + } - return record - } catch (error) { - await SequelizeRepository.rollbackTransaction(transaction) + // find by url + const existing = await EagleEyeContentRepository.findByUrl(data.url, this.options) - SequelizeRepository.handleUniqueFieldError(error, this.options.language, 'EagleEyeContent') + let record - throw error + if (existing) { + record = await EagleEyeContentRepository.update(existing.id, data, this.options) + } else { + record = await EagleEyeContentRepository.create(data, this.options) } + return record + } async findById(id: string): Promise { From 5260efc7836d30cb9b6d551841479afda5c1777e Mon Sep 17 00:00:00 2001 From: anilb Date: Sun, 5 Feb 2023 21:01:47 +0100 Subject: [PATCH 25/36] formatting, all routing finalized --- .../eagleEyeContent/eagleEyeActionCreate.ts | 11 ++ ...tentUpdate.ts => eagleEyeActionDestroy.ts} | 6 +- .../eagleEyeContent/eagleEyeContentList.ts | 17 --- backend/src/api/eagleEyeContent/index.ts | 10 +- backend/src/database/models/eagleEyeAction.ts | 2 +- .../src/database/models/eagleEyeContent.ts | 3 +- .../eagleEyeActionRepository.test.ts | 12 +- .../eagleEyeContentRepository.test.ts | 55 +++++---- .../repositories/eagleEyeContentRepository.ts | 107 ++++++------------ backend/src/i18n/en.ts | 3 +- backend/src/security/permissions.ts | 10 ++ .../__tests__/eagleEyeContentService.test.ts | 15 +-- backend/src/services/eagleEyeActionService.ts | 96 +++++++++------- .../src/services/eagleEyeContentService.ts | 10 +- backend/src/types/eagleEyeTypes.ts | 36 +++--- 15 files changed, 189 insertions(+), 204 deletions(-) rename backend/src/api/eagleEyeContent/{eagleEyeContentUpdate.ts => eagleEyeActionDestroy.ts} (61%) delete mode 100644 backend/src/api/eagleEyeContent/eagleEyeContentList.ts diff --git a/backend/src/api/eagleEyeContent/eagleEyeActionCreate.ts b/backend/src/api/eagleEyeContent/eagleEyeActionCreate.ts index e69de29bb2..8d25a39212 100644 --- a/backend/src/api/eagleEyeContent/eagleEyeActionCreate.ts +++ b/backend/src/api/eagleEyeContent/eagleEyeActionCreate.ts @@ -0,0 +1,11 @@ +import Permissions from '../../security/permissions' +import EagleEyeActionService from '../../services/eagleEyeActionService' +import PermissionChecker from '../../services/user/permissionChecker' + +export default async (req, res) => { + new PermissionChecker(req).validateHas(Permissions.values.eagleEyeActionCreate) + + const payload = await new EagleEyeActionService(req).create(req.body, req.params.contentId) + + await req.responseHandler.success(req, res, payload) +} diff --git a/backend/src/api/eagleEyeContent/eagleEyeContentUpdate.ts b/backend/src/api/eagleEyeContent/eagleEyeActionDestroy.ts similarity index 61% rename from backend/src/api/eagleEyeContent/eagleEyeContentUpdate.ts rename to backend/src/api/eagleEyeContent/eagleEyeActionDestroy.ts index b1d3c0854e..7684b74174 100644 --- a/backend/src/api/eagleEyeContent/eagleEyeContentUpdate.ts +++ b/backend/src/api/eagleEyeContent/eagleEyeActionDestroy.ts @@ -1,11 +1,11 @@ import Permissions from '../../security/permissions' -import EagleEyeContentService from '../../services/eagleEyeContentService' +import EagleEyeActionService from '../../services/eagleEyeActionService' import PermissionChecker from '../../services/user/permissionChecker' export default async (req, res) => { - new PermissionChecker(req).validateHas(Permissions.values.eagleEyeContentEdit) + new PermissionChecker(req).validateHas(Permissions.values.eagleEyeActionDestroy) - const payload = await new EagleEyeContentService(req).update(req.params.id, req.body) + const payload = await new EagleEyeActionService(req).destroy(req.params.actionId) await req.responseHandler.success(req, res, payload) } diff --git a/backend/src/api/eagleEyeContent/eagleEyeContentList.ts b/backend/src/api/eagleEyeContent/eagleEyeContentList.ts deleted file mode 100644 index 40fd0fb9c9..0000000000 --- a/backend/src/api/eagleEyeContent/eagleEyeContentList.ts +++ /dev/null @@ -1,17 +0,0 @@ -import Permissions from '../../security/permissions' -import track from '../../segment/track' -import EagleEyeContentService from '../../services/eagleEyeContentService' -import PermissionChecker from '../../services/user/permissionChecker' - -export default async (req, res) => { - new PermissionChecker(req).validateHas(Permissions.values.eagleEyeContentRead) - const payload = await new EagleEyeContentService(req).findAndCountAll(req.query) - - if (req.query.filter && Object.keys(req.query.filter).length > 0) { - const platforms = req.query.filter.platforms ? req.query.filter.platforms.split(',') : [] - const nDays = req.query.filter.nDays - track('Eagle Eye Filter', { filter: req.query.filter, platforms, nDays }, { ...req }) - } - - await req.responseHandler.success(req, res, payload) -} diff --git a/backend/src/api/eagleEyeContent/index.ts b/backend/src/api/eagleEyeContent/index.ts index 5cd3220e19..167c11ab32 100644 --- a/backend/src/api/eagleEyeContent/index.ts +++ b/backend/src/api/eagleEyeContent/index.ts @@ -9,11 +9,7 @@ export default (app) => { `/tenant/:tenantId/eagleEyeContent/query`, safeWrap(require('./eagleEyeContentQuery').default), ) - app.put( - `/tenant/:tenantId/eagleEyeContent/:id`, - safeWrap(require('./eagleEyeContentUpdate').default), - ) - app.get(`/tenant/:tenantId/eagleEyeContent`, safeWrap(require('./eagleEyeContentList').default)) + app.get( `/tenant/:tenantId/eagleEyeContent/:id`, safeWrap(require('./eagleEyeContentFind').default), @@ -24,4 +20,8 @@ export default (app) => { safeWrap(require('./eagleEyeActionCreate').default), ) + app.delete( + `/tenant/:tenantId/eagleEyeContent/:contentId/action/:actionId`, + safeWrap(require('./eagleEyeActionDestroy').default), + ) } diff --git a/backend/src/database/models/eagleEyeAction.ts b/backend/src/database/models/eagleEyeAction.ts index 14eaab4260..116b069fb2 100644 --- a/backend/src/database/models/eagleEyeAction.ts +++ b/backend/src/database/models/eagleEyeAction.ts @@ -38,7 +38,7 @@ export default (sequelize) => { }) models.eagleEyeAction.belongsTo(models.eagleEyeContent, { - as: 'content', + as: 'content', }) } diff --git a/backend/src/database/models/eagleEyeContent.ts b/backend/src/database/models/eagleEyeContent.ts index 198f28e754..9b00bc070c 100644 --- a/backend/src/database/models/eagleEyeContent.ts +++ b/backend/src/database/models/eagleEyeContent.ts @@ -15,7 +15,7 @@ const eagleEyeContentModel = { }, post: { type: DataTypes.JSONB, - allowNull: false + allowNull: false, }, url: { type: DataTypes.TEXT, @@ -47,7 +47,6 @@ export default (sequelize) => { as: 'actions', foreignKey: 'contentId', }) - } return eagleEyeContent diff --git a/backend/src/database/repositories/__tests__/eagleEyeActionRepository.test.ts b/backend/src/database/repositories/__tests__/eagleEyeActionRepository.test.ts index f3e035cc20..f3a64fa6ec 100644 --- a/backend/src/database/repositories/__tests__/eagleEyeActionRepository.test.ts +++ b/backend/src/database/repositories/__tests__/eagleEyeActionRepository.test.ts @@ -33,13 +33,16 @@ describe('eagleEyeActionRepository tests', () => { const contentCreated = await EagleEyeContentRepository.create(content, mockIRepositoryOptions) - - const action: EagleEyeAction= { + const action: EagleEyeAction = { type: EagleEyeActionType.BOOKMARK, timestamp: '2022-07-27T19:13:30Z', - } + } - const actionCreated = await EagleEyeActionRepository.createActionForContent(action, contentCreated.id, mockIRepositoryOptions) + const actionCreated = await EagleEyeActionRepository.createActionForContent( + action, + contentCreated.id, + mockIRepositoryOptions, + ) actionCreated.createdAt = (actionCreated.createdAt as Date).toISOString().split('T')[0] actionCreated.updatedAt = (actionCreated.updatedAt as Date).toISOString().split('T')[0] @@ -56,6 +59,5 @@ describe('eagleEyeActionRepository tests', () => { } expect(expectedAction).toStrictEqual(actionCreated) }) - }) }) diff --git a/backend/src/database/repositories/__tests__/eagleEyeContentRepository.test.ts b/backend/src/database/repositories/__tests__/eagleEyeContentRepository.test.ts index eef92a3679..96666a7148 100644 --- a/backend/src/database/repositories/__tests__/eagleEyeContentRepository.test.ts +++ b/backend/src/database/repositories/__tests__/eagleEyeContentRepository.test.ts @@ -3,8 +3,6 @@ import SequelizeTestUtils from '../../utils/sequelizeTestUtils' import { EagleEyeActionType, EagleEyeContent } from '../../../types/eagleEyeTypes' import EagleEyeActionRepository from '../eagleEyeActionRepository' - - const db = null describe('eagleEyeContentRepository tests', () => { @@ -422,17 +420,17 @@ describe('eagleEyeContentRepository tests', () => { describe('findAndCountAll method', () => { it('Should find eagle eye contant, various cases', async () => { - // create random tenant with one user + // create random tenant with one user const mockIRepositoryOptions = await SequelizeTestUtils.getTestIRepositoryOptions(db) // create additional users for same tenant to test out actionBy filtering const randomUser = await SequelizeTestUtils.getRandomUser() - console.log("random user: ") + console.log('random user: ') console.log(randomUser) const user2 = await mockIRepositoryOptions.database.user.create(randomUser) - + await mockIRepositoryOptions.database.tenantUser.create({ roles: ['admin'], status: 'active', @@ -456,7 +454,6 @@ describe('eagleEyeContentRepository tests', () => { mockIRepositoryOptions, ) - // one with a bookmark action let c2 = await EagleEyeContentRepository.create( { @@ -486,7 +483,7 @@ describe('eagleEyeContentRepository tests', () => { // another content with a thumbs-up(user1) and a bookmark(user2) action let c3 = await EagleEyeContentRepository.create( - { + { platform: 'devto', url: 'https://some-devto-url', post: { @@ -498,7 +495,7 @@ describe('eagleEyeContentRepository tests', () => { }, mockIRepositoryOptions, ) - + // add the thumbs up action await EagleEyeActionRepository.createActionForContent( { @@ -516,38 +513,40 @@ describe('eagleEyeContentRepository tests', () => { timestamp: '2022-09-30T23:11:10Z', }, c3.id, - {...mockIRepositoryOptions, currentUser: user2}, + { ...mockIRepositoryOptions, currentUser: user2 }, ) c3 = await EagleEyeContentRepository.findById(c3.id, mockIRepositoryOptions) - // filter by action type - let res = await EagleEyeContentRepository.findAndCountAll({ - advancedFilter: { - action: { - type: EagleEyeActionType.BOOKMARK - - } - } - }, mockIRepositoryOptions) - + let res = await EagleEyeContentRepository.findAndCountAll( + { + advancedFilter: { + action: { + type: EagleEyeActionType.BOOKMARK, + }, + }, + }, + mockIRepositoryOptions, + ) expect(res.count).toBe(2) - expect(res.rows.sort((a, b) => (a.createdAt > b.createdAt ? 1 : -1))).toStrictEqual([c2,c3]) + expect(res.rows.sort((a, b) => (a.createdAt > b.createdAt ? 1 : -1))).toStrictEqual([c2, c3]) // filter by actionBy - res = await EagleEyeContentRepository.findAndCountAll({ - advancedFilter: { - action: { - actionById: user2.id - } - } - }, mockIRepositoryOptions) + res = await EagleEyeContentRepository.findAndCountAll( + { + advancedFilter: { + action: { + actionById: user2.id, + }, + }, + }, + mockIRepositoryOptions, + ) expect(res.count).toBe(1) expect(res.rows).toStrictEqual([c3]) - }) }) }) diff --git a/backend/src/database/repositories/eagleEyeContentRepository.ts b/backend/src/database/repositories/eagleEyeContentRepository.ts index 66b87609f6..9a34d4d9d4 100644 --- a/backend/src/database/repositories/eagleEyeContentRepository.ts +++ b/backend/src/database/repositories/eagleEyeContentRepository.ts @@ -9,33 +9,26 @@ import { QueryOutput } from './filters/queryTypes' import EagleEyeActionRepository from './eagleEyeActionRepository' export default class EagleEyeContentRepository { - - static async create(data:EagleEyeContent, options:IRepositoryOptions): Promise{ + static async create( + data: EagleEyeContent, + options: IRepositoryOptions, + ): Promise { const currentTenant = SequelizeRepository.getCurrentTenant(options) - const record = await options.database.eagleEyeContent.create( - { - ...lodash.pick(data, [ - 'platform', - 'post', - 'url', - 'postedAt' - ]), - tenantId: currentTenant.id, - }, - ) + const record = await options.database.eagleEyeContent.create({ + ...lodash.pick(data, ['platform', 'post', 'url', 'postedAt']), + tenantId: currentTenant.id, + }) - if (data.actions){ - for (const action of data.actions){ + if (data.actions) { + for (const action of data.actions) { await EagleEyeActionRepository.createActionForContent(action, record.id, options) } } return this.findById(record.id, options) - } - static async update(id, data, options: IRepositoryOptions) { const currentUser = SequelizeRepository.getCurrentUser(options) @@ -55,15 +48,9 @@ export default class EagleEyeContentRepository { throw new Error404() } - record = await record.update( { - ...lodash.pick(data, [ - 'platform', - 'post', - 'postedAt', - 'url', - ]), + ...lodash.pick(data, ['platform', 'post', 'postedAt', 'url']), updatedById: currentUser.id, }, { @@ -74,13 +61,15 @@ export default class EagleEyeContentRepository { return this.findById(record.id, options) } - static async findById(id:string, options: IRepositoryOptions) { + static async findById(id: string, options: IRepositoryOptions) { const transaction = SequelizeRepository.getTransaction(options) - const include = [{ - model: options.database.eagleEyeAction, - as: 'actions', - }] + const include = [ + { + model: options.database.eagleEyeAction, + as: 'actions', + }, + ] const currentTenant = SequelizeRepository.getCurrentTenant(options) @@ -122,30 +111,17 @@ export default class EagleEyeContentRepository { } static async findAndCountAll( - { - advancedFilter = null as any, - limit = 0, - offset = 0, - orderBy = '', - }, + { advancedFilter = null as any, limit = 0, offset = 0, orderBy = '' }, options: IRepositoryOptions, ) { - const actionsSequelizeInclude = { model: options.database.eagleEyeAction, as: 'actions', where: {}, - } - if (advancedFilter && advancedFilter.action) { - - const actionQueryParser = new QueryParser( - { - }, - options, - ) + const actionQueryParser = new QueryParser({}, options) const parsedActionQuery: QueryOutput = actionQueryParser.parse({ filter: advancedFilter.action, @@ -156,15 +132,9 @@ export default class EagleEyeContentRepository { delete advancedFilter.action } - const include = [ - actionsSequelizeInclude, - ] - - const contentParser = new QueryParser( - {}, - options, - ) + const include = [actionsSequelizeInclude] + const contentParser = new QueryParser({}, options) const parsed: QueryOutput = contentParser.parse({ filter: advancedFilter, @@ -173,7 +143,6 @@ export default class EagleEyeContentRepository { offset, }) - let { rows, count, // eslint-disable-line prefer-const @@ -184,34 +153,32 @@ export default class EagleEyeContentRepository { limit: parsed.limit, offset: parsed.offset, transaction: SequelizeRepository.getTransaction(options), - subQuery: false + subQuery: false, }) // If we have an actions filter, we should query again to eager - // load the all actions on a content because previous query will + // load the all actions on a content because previous query will // omit actions that don't match the given action filter - if (Object.keys(actionsSequelizeInclude.where).length !== 0) - { - rows = (await options.database.eagleEyeContent.findAndCountAll({ - include: [{...actionsSequelizeInclude, where: {}}], - where: { id: { [Op.in]: rows.map( (i) => i.id) } }, - order: parsed.order, - limit: parsed.limit, - offset: parsed.offset, - transaction: SequelizeRepository.getTransaction(options), - subQuery: false - - })).rows + if (Object.keys(actionsSequelizeInclude.where).length !== 0) { + rows = ( + await options.database.eagleEyeContent.findAndCountAll({ + include: [{ ...actionsSequelizeInclude, where: {} }], + where: { id: { [Op.in]: rows.map((i) => i.id) } }, + order: parsed.order, + limit: parsed.limit, + offset: parsed.offset, + transaction: SequelizeRepository.getTransaction(options), + subQuery: false, + }) + ).rows } - rows = await this._populateRelationsForRows(rows) return { rows, count, limit: parsed.limit, offset: parsed.offset } - } - static async findByUrl(url:string, options: IRepositoryOptions) { + static async findByUrl(url: string, options: IRepositoryOptions) { const transaction = SequelizeRepository.getTransaction(options) const include = [] diff --git a/backend/src/i18n/en.ts b/backend/src/i18n/en.ts index 0c4a89b705..82896eb36d 100644 --- a/backend/src/i18n/en.ts +++ b/backend/src/i18n/en.ts @@ -109,7 +109,8 @@ const en = { message: 'Project not found', }, eagleEye: { - urlRequiredWhenUpserting: 'URL field is mandatory when upserting eagleEyeContent' + urlRequiredWhenUpserting: 'URL field is mandatory when upserting eagleEyeContent', + contentNotFound: 'Eagle eye content not found. Action will not be created.', }, integrations: { badEndpoint: 'Bad endpoint: {0}', diff --git a/backend/src/security/permissions.ts b/backend/src/security/permissions.ts index 74482a3f03..ec28c71024 100644 --- a/backend/src/security/permissions.ts +++ b/backend/src/security/permissions.ts @@ -422,6 +422,16 @@ class Permissions { allowedRoles: [roles.admin, roles.readonly], allowedPlans: [plans.essential, plans.growth], }, + eagleEyeActionCreate: { + id: 'eagleEyeActionCreate', + allowedRoles: [roles.admin], + allowedPlans: [plans.growth], + }, + eagleEyeActionDestroy: { + id: 'eagleEyeActionDestroy', + allowedRoles: [roles.admin], + allowedPlans: [plans.growth], + }, eagleEyeContentCreate: { id: 'eagleEyeContentCreate', allowedRoles: [roles.admin], diff --git a/backend/src/services/__tests__/eagleEyeContentService.test.ts b/backend/src/services/__tests__/eagleEyeContentService.test.ts index f67db5118e..9170b4a645 100644 --- a/backend/src/services/__tests__/eagleEyeContentService.test.ts +++ b/backend/src/services/__tests__/eagleEyeContentService.test.ts @@ -27,10 +27,12 @@ describe('EagleEyeContentService tests', () => { }, postedAt: '2020-05-27T15:13:30Z', tenantId: mockIRepositoryOptions.currentTenant.id, - actions: [{ - type: EagleEyeActionType.BOOKMARK, - timestamp: '2022-06-27T14:13:30Z', - }] + actions: [ + { + type: EagleEyeActionType.BOOKMARK, + timestamp: '2022-06-27T14:13:30Z', + }, + ], } const service = new EagleEyeContentService(mockIRepositoryOptions) @@ -60,11 +62,6 @@ describe('EagleEyeContentService tests', () => { expect(contents.rows).toStrictEqual([c1Upserted]) expect(c1Upserted.id).toEqual(c1.id) expect(contents.rows[0].post).toStrictEqual(contentWithSameUrl.post) - }) - - }) - - }) diff --git a/backend/src/services/eagleEyeActionService.ts b/backend/src/services/eagleEyeActionService.ts index 7711915543..75b71c9178 100644 --- a/backend/src/services/eagleEyeActionService.ts +++ b/backend/src/services/eagleEyeActionService.ts @@ -1,5 +1,6 @@ import EagleEyeActionRepository from '../database/repositories/eagleEyeActionRepository' import EagleEyeContentRepository from '../database/repositories/eagleEyeContentRepository' +import SequelizeRepository from '../database/repositories/sequelizeRepository' import Error404 from '../errors/Error404' import { EagleEyeAction, EagleEyeActionType } from '../types/eagleEyeTypes' import { IServiceOptions } from './IServiceOptions' @@ -14,54 +15,71 @@ export default class EagleEyeActionService extends LoggingBase { } async create(data: EagleEyeAction, contentId: string): Promise { + const transaction = await SequelizeRepository.createTransaction(this.options) + // find content const content = await EagleEyeContentRepository.findById(contentId, this.options) if (!content) { - throw new Error404('Content not found..') + throw new Error404(this.options.language, 'errors.eagleEye.contentNotFound') } - const existingUserActions: EagleEyeAction[] = content.actions - .filter((a) => a.actionById === this.options.currentUser.id) + const existingUserActions: EagleEyeAction[] = content.actions.filter( + (a) => a.actionById === this.options.currentUser.id, + ) const existingUserActionTypes = existingUserActions.map((a) => a.type) - // check if already bookmarked - if yes ignore the new action and return existing - if ( - data.type === EagleEyeActionType.BOOKMARK && - existingUserActionTypes.includes(EagleEyeActionType.BOOKMARK) - ) { - return existingUserActions.find( (a) => a.type === EagleEyeActionType.BOOKMARK) - } - - // thumbs up and down should be mutually exclusive - if ( - data.type === EagleEyeActionType.THUMBS_DOWN && - existingUserActionTypes.includes(EagleEyeActionType.THUMBS_UP) - ) { - await EagleEyeActionRepository.removeActionFromContent( - EagleEyeActionType.THUMBS_UP, - contentId, - this.options, - ) - } else if ( - data.type === EagleEyeActionType.THUMBS_UP && - existingUserActionTypes.includes(EagleEyeActionType.THUMBS_DOWN) - ) { - await EagleEyeActionRepository.removeActionFromContent( - EagleEyeActionType.THUMBS_DOWN, - contentId, - this.options, - ) - } - - // add new action - return EagleEyeActionRepository.createActionForContent(data, contentId, this.options) + try { + // check if already bookmarked - if yes ignore the new action and return existing + if ( + data.type === EagleEyeActionType.BOOKMARK && + existingUserActionTypes.includes(EagleEyeActionType.BOOKMARK) + ) { + return existingUserActions.find((a) => a.type === EagleEyeActionType.BOOKMARK) + } + + // thumbs up and down should be mutually exclusive + if ( + data.type === EagleEyeActionType.THUMBS_DOWN && + existingUserActionTypes.includes(EagleEyeActionType.THUMBS_UP) + ) { + await EagleEyeActionRepository.removeActionFromContent( + EagleEyeActionType.THUMBS_UP, + contentId, + this.options, + ) + } else if ( + data.type === EagleEyeActionType.THUMBS_UP && + existingUserActionTypes.includes(EagleEyeActionType.THUMBS_DOWN) + ) { + await EagleEyeActionRepository.removeActionFromContent( + EagleEyeActionType.THUMBS_DOWN, + contentId, + this.options, + ) + } + + // add new action + const record = await EagleEyeActionRepository.createActionForContent( + data, + contentId, + this.options, + ) + + await SequelizeRepository.commitTransaction(transaction) + + return record + } catch (error) { + await SequelizeRepository.rollbackTransaction(transaction) + + SequelizeRepository.handleUniqueFieldError(error, this.options.language, 'EagleEyeContent') + + throw error + } } - - async destroy(id: string){ - + async destroy(id: string) { const action = await EagleEyeActionRepository.findById(id, this.options) const contentId = action.contentId @@ -72,8 +90,8 @@ export default class EagleEyeActionService extends LoggingBase { const content = await EagleEyeContentRepository.findById(contentId, this.options) // if content no longer has any actions attached to it, also delete the content - if (content.actions.length === 0){ - await EagleEyeContentRepository.destroy(contentId, this.options) + if (content.actions.length === 0) { + await EagleEyeContentRepository.destroy(contentId, this.options) } } } diff --git a/backend/src/services/eagleEyeContentService.ts b/backend/src/services/eagleEyeContentService.ts index fdbd133d55..4a58b4f766 100644 --- a/backend/src/services/eagleEyeContentService.ts +++ b/backend/src/services/eagleEyeContentService.ts @@ -24,7 +24,6 @@ export default class EagleEyeContentService extends LoggingBase { * @returns Created EagleEyeContent record. */ async upsert(data: EagleEyeContent): Promise { - if (!data.url) { throw new Error400(this.options.language, 'errors.eagleEye.urlRequiredWhenUpserting') } @@ -41,7 +40,6 @@ export default class EagleEyeContentService extends LoggingBase { } return record - } async findById(id: string): Promise { @@ -59,9 +57,11 @@ export default class EagleEyeContentService extends LoggingBase { ) } - /** TODO + /** + TODO + */ + /* eslint-disable-next-line */ async search(args) { - + return null } - */ } diff --git a/backend/src/types/eagleEyeTypes.ts b/backend/src/types/eagleEyeTypes.ts index 556ed1fd8c..2aa5a9bd1d 100644 --- a/backend/src/types/eagleEyeTypes.ts +++ b/backend/src/types/eagleEyeTypes.ts @@ -1,27 +1,25 @@ export enum EagleEyeActionType { - THUMBS_UP = 'thumbs-up', - THUMBS_DOWN = 'thumbs-down', - BOOKMARK = 'bookmark', + THUMBS_UP = 'thumbs-up', + THUMBS_DOWN = 'thumbs-down', + BOOKMARK = 'bookmark', } - export interface EagleEyeAction { - id?: string - type: EagleEyeActionType - timestamp: Date | string - createdAt?: Date | string - updatedAt?: Date | string + id?: string + type: EagleEyeActionType + timestamp: Date | string + createdAt?: Date | string + updatedAt?: Date | string } export interface EagleEyeContent { - id?: string - platform: string - post: any - url: string - actions?: EagleEyeAction[] - tenantId: string - postedAt: string - createdAt?: Date | string - updatedAt?: Date | string + id?: string + platform: string + post: any + url: string + actions?: EagleEyeAction[] + tenantId: string + postedAt: string + createdAt?: Date | string + updatedAt?: Date | string } - From 127c10b25bb0fb49b5cc873a55111d904e739226 Mon Sep 17 00:00:00 2001 From: anilb Date: Sun, 5 Feb 2023 21:07:17 +0100 Subject: [PATCH 26/36] removed un-used get posts by keywords file --- .../usecases/hackerNews/getPostsByKeywords.ts | 27 ------------------- 1 file changed, 27 deletions(-) delete mode 100644 backend/src/serverless/integrations/usecases/hackerNews/getPostsByKeywords.ts diff --git a/backend/src/serverless/integrations/usecases/hackerNews/getPostsByKeywords.ts b/backend/src/serverless/integrations/usecases/hackerNews/getPostsByKeywords.ts deleted file mode 100644 index 0e19a3ac5e..0000000000 --- a/backend/src/serverless/integrations/usecases/hackerNews/getPostsByKeywords.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { IServiceOptions } from '../../../../services/IServiceOptions' -import { EagleEyeResponses, EagleEyeInput } from '../../types/hackerNewsTypes' -import { Logger } from '../../../../utils/logging' -import { timeout } from '../../../../utils/timing' -import EagleEyeContentService from '../../../../services/eagleEyeContentService' - -async function getPostsByKeyword( - input: EagleEyeInput, - options: IServiceOptions, - logger: Logger, -): Promise { - await timeout(2000) - - try { - const eagleEyeService = new EagleEyeContentService(options) - return await eagleEyeService.keywordMatch({ - keywords: input.keywords, - nDays: input.nDays, - platform: 'hacker_news', - }) - } catch (err) { - logger.error({ err, input }, 'Error while getting posts by keyword in EagleEye') - throw err - } -} - -export default getPostsByKeyword From 7361d5917a1e8ebc785fa2b6f176ce7ed9ea6381 Mon Sep 17 00:00:00 2001 From: anilb Date: Sun, 5 Feb 2023 21:33:54 +0100 Subject: [PATCH 27/36] getPostsByKeyword re-added bcs it's being used by hackernews --- .../usecases/hackerNews/getPostsByKeywords.ts | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 backend/src/serverless/integrations/usecases/hackerNews/getPostsByKeywords.ts diff --git a/backend/src/serverless/integrations/usecases/hackerNews/getPostsByKeywords.ts b/backend/src/serverless/integrations/usecases/hackerNews/getPostsByKeywords.ts new file mode 100644 index 0000000000..7d5a837546 --- /dev/null +++ b/backend/src/serverless/integrations/usecases/hackerNews/getPostsByKeywords.ts @@ -0,0 +1,27 @@ +import { IServiceOptions } from '../../../../services/IServiceOptions' +import { EagleEyeResponses, EagleEyeInput } from '../../types/hackerNewsTypes' +import { Logger } from '../../../../utils/logging' +import { timeout } from '../../../../utils/timing' +import EagleEyeContentService from '../../../../services/eagleEyeContentService' + +async function getPostsByKeyword( + input: EagleEyeInput, + options: IServiceOptions, + logger: Logger, +): Promise { + await timeout(2000) + + try { + const eagleEyeService = new EagleEyeContentService(options) + return await eagleEyeService.keywordMatch({ + keywords: input.keywords, + nDays: input.nDays, + platform: 'hacker_news', + }) + } catch (err) { + logger.error({ err, input }, 'Error while getting posts by keyword in EagleEye') + throw err + } +} + +export default getPostsByKeyword \ No newline at end of file From 674d3af0e9481c2f8df6a7e73582dafd780de970 Mon Sep 17 00:00:00 2001 From: Joan Reyero Date: Tue, 7 Feb 2023 13:36:17 +0100 Subject: [PATCH 28/36] Added endpoint for EagleEye settings --- .../eagleEyeContent/eagleEyeSettingsUpdate.ts | 11 ++ backend/src/api/eagleEyeContent/index.ts | 5 + .../U1675702339__eagleEyeSettings.sql | 2 + .../V1675702339__eagleEyeSettings.sql | 2 + backend/src/database/models/user.ts | 7 + .../database/repositories/userRepository.ts | 31 ++++ backend/src/i18n/en.ts | 12 ++ .../src/services/eagleEyeSettingsService.ts | 144 ++++++++++++++++++ backend/src/types/eagleEyeTypes.ts | 47 ++++++ 9 files changed, 261 insertions(+) create mode 100644 backend/src/api/eagleEyeContent/eagleEyeSettingsUpdate.ts create mode 100644 backend/src/database/migrations/U1675702339__eagleEyeSettings.sql create mode 100644 backend/src/database/migrations/V1675702339__eagleEyeSettings.sql create mode 100644 backend/src/services/eagleEyeSettingsService.ts diff --git a/backend/src/api/eagleEyeContent/eagleEyeSettingsUpdate.ts b/backend/src/api/eagleEyeContent/eagleEyeSettingsUpdate.ts new file mode 100644 index 0000000000..635e4a7411 --- /dev/null +++ b/backend/src/api/eagleEyeContent/eagleEyeSettingsUpdate.ts @@ -0,0 +1,11 @@ +import Permissions from '../../security/permissions' +import EagleEyeSettingsService from '../../services/eagleEyeSettingsService' +import PermissionChecker from '../../services/user/permissionChecker' + +export default async (req, res) => { + new PermissionChecker(req).validateHas(Permissions.values.eagleEyeActionCreate) + + const payload = await new EagleEyeSettingsService(req).update(req.body) + + await req.responseHandler.success(req, res, payload) +} diff --git a/backend/src/api/eagleEyeContent/index.ts b/backend/src/api/eagleEyeContent/index.ts index 167c11ab32..cbc3f21db4 100644 --- a/backend/src/api/eagleEyeContent/index.ts +++ b/backend/src/api/eagleEyeContent/index.ts @@ -20,6 +20,11 @@ export default (app) => { safeWrap(require('./eagleEyeActionCreate').default), ) + app.put( + `/tenant/:tenantId/eagleEyeContent/settings`, + safeWrap(require('./eagleEyeSettingsUpdate').default), + ) + app.delete( `/tenant/:tenantId/eagleEyeContent/:contentId/action/:actionId`, safeWrap(require('./eagleEyeActionDestroy').default), diff --git a/backend/src/database/migrations/U1675702339__eagleEyeSettings.sql b/backend/src/database/migrations/U1675702339__eagleEyeSettings.sql new file mode 100644 index 0000000000..e3a0f4fe1e --- /dev/null +++ b/backend/src/database/migrations/U1675702339__eagleEyeSettings.sql @@ -0,0 +1,2 @@ +ALTER TABLE public."users" +DROP COLUMN "eagleEyeSettings"; \ No newline at end of file diff --git a/backend/src/database/migrations/V1675702339__eagleEyeSettings.sql b/backend/src/database/migrations/V1675702339__eagleEyeSettings.sql new file mode 100644 index 0000000000..7e8e46fbf2 --- /dev/null +++ b/backend/src/database/migrations/V1675702339__eagleEyeSettings.sql @@ -0,0 +1,2 @@ +ALTER TABLE public."users" +ADD COLUMN "eagleEyeSettings" JSONB DEFAULT '{"onboarded": false}'; diff --git a/backend/src/database/models/user.ts b/backend/src/database/models/user.ts index a64482691b..3e9990465e 100644 --- a/backend/src/database/models/user.ts +++ b/backend/src/database/models/user.ts @@ -102,6 +102,13 @@ export default (sequelize, DataTypes) => { len: [0, 255], }, }, + eagleEyeSettings: { + type: DataTypes.JSONB, + allowNull: false, + defaultValue: { + onboarded: false, + }, + }, }, { indexes: [ diff --git a/backend/src/database/repositories/userRepository.ts b/backend/src/database/repositories/userRepository.ts index 8b80aee5ed..781e93d1cf 100644 --- a/backend/src/database/repositories/userRepository.ts +++ b/backend/src/database/repositories/userRepository.ts @@ -320,6 +320,37 @@ export default class UserRepository { return this.findById(user.id, options) } + static async updateEagleEyeSettings(id: string, data, options: IRepositoryOptions) { + const currentUser = SequelizeRepository.getCurrentUser(options) + const transaction = SequelizeRepository.getTransaction(options) + + const user = await options.database.user.findByPk(id, { + transaction, + }) + + await user.update( + { + eagleEyeSettings: data, + updatedById: currentUser.id, + }, + { transaction }, + ) + + await AuditLogRepository.log( + { + entityName: 'user', + entityId: user.id, + action: AuditLogRepository.UPDATE, + values: { + ...user.get({ plain: true }), + eagleEyeSettings: data, + }, + }, + options, + ) + return this.findById(user.id, options) + } + static async findByEmail(email, options: IRepositoryOptions) { const transaction = SequelizeRepository.getTransaction(options) diff --git a/backend/src/i18n/en.ts b/backend/src/i18n/en.ts index 82896eb36d..f0fad60032 100644 --- a/backend/src/i18n/en.ts +++ b/backend/src/i18n/en.ts @@ -111,6 +111,18 @@ const en = { eagleEye: { urlRequiredWhenUpserting: 'URL field is mandatory when upserting eagleEyeContent', contentNotFound: 'Eagle eye content not found. Action will not be created.', + feedSettingsMissing: 'Feed settings are missing. Settings not updated.', + keywordsMissing: + 'Either keywords or exactKeywords are required in feeds. Settings not updated.', + platformMissing: + 'feed.platforms is required and must be a non-empty list. Settings not updated.', + platformInvalid: `feed.platforms contains {0}, which is not in [{1}]. Settings not updated.`, + publishedDateMissing: + 'feed.publishedDate is missing or invalid. It should be one of [{0}]. Settings not updated.', + emailInvalid: 'emailDigest.email needs a valid email address. Settings not updated.', + frequencyInvalid: + 'emailDigest.frequency needs to be one of daily, weekly. Settings not updated.', + timeInvalid: 'emailDigest.time needs to be a valid time. Settings not updated.', }, integrations: { badEndpoint: 'Bad endpoint: {0}', diff --git a/backend/src/services/eagleEyeSettingsService.ts b/backend/src/services/eagleEyeSettingsService.ts new file mode 100644 index 0000000000..cd6826cf29 --- /dev/null +++ b/backend/src/services/eagleEyeSettingsService.ts @@ -0,0 +1,144 @@ +import moment from 'moment' +import lodash from 'lodash' +import SequelizeRepository from '../database/repositories/sequelizeRepository' +import UserRepository from '../database/repositories/userRepository' +import Error400 from '../errors/Error400' +import { + EagleEyeSettings, + EagleEyeFeedSettings, + EagleEyePlatforms, + EagleEyePublishedDates, + EagleEyeEmailDigestSettings, +} from '../types/eagleEyeTypes' +import { IServiceOptions } from './IServiceOptions' +import { LoggingBase } from './loggingBase' + +export default class EagleEyeSettingsService extends LoggingBase { + options: IServiceOptions + + constructor(options) { + super(options) + this.options = options + } + + getFeed(data: EagleEyeFeedSettings) { + if (!data) { + throw new Error400(this.options.language, 'errors.eagleEye.feedSettingsMissing') + } + + if (!data.keywords && !data.exactKeywords) { + throw new Error400(this.options.language, 'errors.eagleEye.keywordsMissing') + } + + if (!data.platforms || data.platforms.length === 0) { + throw new Error400(this.options.language, 'errors.eagleEye.platformMissing') + } + + const platforms = Object.values(EagleEyePlatforms) as string[] + data.platforms.forEach((platform) => { + if (!platforms.includes(platform)) { + throw new Error400( + this.options.language, + 'errors.eagleEye.platformInvalid', + platform, + platforms.join(', '), + ) + } + }) + + const publishedDates = Object.values(EagleEyePublishedDates) as string[] + if (publishedDates.indexOf(data.publishedDate as string) === -1) { + throw new Error400( + this.options.language, + 'errors.eagleEye.publishedDateMissing', + publishedDates.join(', '), + ) + } + + data.publishedDate = EagleEyeSettingsService.switchDate(data.publishedDate as string) + return lodash.pick(data, [ + 'keywords', + 'exactKeywords', + 'excludedKeywords', + 'publishedDate', + 'platforms', + ]) + } + + static switchDate(date: string) { + switch (date) { + case 'Last 24h': + return moment().subtract(1, 'days').format('YYYY-MM-DD') + case 'Last 7 days': + return moment().subtract(7, 'days').format('YYYY-MM-DD') + case 'Last 14 days': + return moment().subtract(14, 'days').format('YYYY-MM-DD') + case 'Last 30 days': + return moment().subtract(30, 'days').format('YYYY-MM-DD') + case 'Last 90 days': + return moment().subtract(90, 'days').format('YYYY-MM-DD') + default: + return null + } + } + + getEmailDigestSettings(data: EagleEyeEmailDigestSettings, feed: EagleEyeFeedSettings) { + if (!data.matchFeedSettings) { + data.feed = this.getFeed(data.feed) + } else { + data.feed = feed + } + + const emailRegex = + /^(([^<>()[\]\\.,;:\s@"]+(\.[^<>()[\]\\.,;:\s@"]+)*)|(".+"))@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\])|(([a-zA-Z\-0-9]+\.)+[a-zA-Z]{2,}))$/ + if (!emailRegex.test(data.email)) { + throw new Error400(this.options.language, 'errors.eagleEye.emailInvalid') + } + + if (['daily', 'weekly'].indexOf(data.frequency) === -1) { + throw new Error400(this.options.language, 'errors.eagleEye.frequencyInvalid') + } + + const timeRegex = /^([0-1]?[0-9]|2[0-3]):[0-5][0-9]?$/ + if (!timeRegex.test(data.time)) { + throw new Error400(this.options.language, 'errors.eagleEye.timeInvalid') + } + + return lodash.pick(data, ['email', 'frequency', 'time', 'matchFeedSettings', 'feed']) + } + + async update(data: EagleEyeSettings): Promise { + const transaction = await SequelizeRepository.createTransaction(this.options) + try { + data.onboarded = true + + data.feed = this.getFeed(data.feed) + + if (data.emailDigest || data.emailDigestActive) { + data.emailDigestActive = true + + data.emailDigest = this.getEmailDigestSettings(data.emailDigest, data.feed) + } else { + data.emailDigestActive = false + } + + data = lodash.pick(data, ['onboarded', 'feed', 'emailDigestActive', 'emailDigest']) + + const userOut = await UserRepository.updateEagleEyeSettings( + this.options.currentUser.id, + { eagleEyeSettings: data }, + { ...this.options, transaction }, + ) + + await SequelizeRepository.commitTransaction(transaction) + + return userOut.eagleEyeSettings + } catch (error) { + await SequelizeRepository.rollbackTransaction(transaction) + + SequelizeRepository.handleUniqueFieldError(error, this.options.language, 'EagleEyeContent') + + throw error + } + } +} diff --git a/backend/src/types/eagleEyeTypes.ts b/backend/src/types/eagleEyeTypes.ts index 2aa5a9bd1d..2f1f2d4d59 100644 --- a/backend/src/types/eagleEyeTypes.ts +++ b/backend/src/types/eagleEyeTypes.ts @@ -23,3 +23,50 @@ export interface EagleEyeContent { createdAt?: Date | string updatedAt?: Date | string } + +export interface EagleEyeFeedSettings { + keywords: string[] + exactKeywords: string[] + excludedKeywords: string[] + publishedDate: string | Date + platforms: string[] +} + +export interface EagleEyeEmailDigestSettings { + email: string + frequency: 'daily' | 'weekly' + time: string + feed: EagleEyeFeedSettings + matchFeedSettings: boolean +} + +export interface EagleEyeSettings { + onboarded: boolean + feed: EagleEyeFeedSettings + emailDigestActive: boolean + emailDigest?: EagleEyeEmailDigestSettings +} + +// Enum for EagleEyePlatforms +export enum EagleEyePlatforms { + GITHUB = 'github', + HACKERNEWS = 'hackernews', + DEVTO = 'devto', + REDDIT = 'reddit', + MEDIUM = 'medium', + STACKOVERFLOW = 'stackoverflow', + TWITTER = 'twitter', + YOUTUBE = 'youtube', + PRODUCTHUNT = 'producthunt', + KAGGLE = 'kaggle', + HASHNODE = 'hashnode', + LINKEDIN = 'linkedin', +} + +export enum EagleEyePublishedDates { + LAST_24_HOURS = 'Last 24h', + LAST_7_DAYS = 'Last 7 days', + LAST_14_DAYS = 'Last 14 days', + LAST_30_DAYS = 'Last 30 days', + LAST_90_DAYS = 'Last 90 days', +} From 3cb682c5f55e7fcf145c074380726bcd805cf4a8 Mon Sep 17 00:00:00 2001 From: Joan Reyero Date: Tue, 7 Feb 2023 13:55:19 +0100 Subject: [PATCH 29/36] Added comments --- .../usecases/hackerNews/getPostsByKeywords.ts | 2 +- .../src/services/eagleEyeSettingsService.ts | 42 ++++++++++++++++++- 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/backend/src/serverless/integrations/usecases/hackerNews/getPostsByKeywords.ts b/backend/src/serverless/integrations/usecases/hackerNews/getPostsByKeywords.ts index 7d5a837546..0e19a3ac5e 100644 --- a/backend/src/serverless/integrations/usecases/hackerNews/getPostsByKeywords.ts +++ b/backend/src/serverless/integrations/usecases/hackerNews/getPostsByKeywords.ts @@ -24,4 +24,4 @@ async function getPostsByKeyword( } } -export default getPostsByKeyword \ No newline at end of file +export default getPostsByKeyword diff --git a/backend/src/services/eagleEyeSettingsService.ts b/backend/src/services/eagleEyeSettingsService.ts index cd6826cf29..d500a8ff40 100644 --- a/backend/src/services/eagleEyeSettingsService.ts +++ b/backend/src/services/eagleEyeSettingsService.ts @@ -21,19 +21,28 @@ export default class EagleEyeSettingsService extends LoggingBase { this.options = options } + /** + * Validate and normalize feed settings. + * @param data Feed data of type EagleEyeFeedSettings + * @returns Normalized feed data if the input is valid. Otherwise a 400 Error + */ getFeed(data: EagleEyeFeedSettings) { + // Feed is compulsory if (!data) { throw new Error400(this.options.language, 'errors.eagleEye.feedSettingsMissing') } + // We need at least one of keywords or exactKeywords if (!data.keywords && !data.exactKeywords) { throw new Error400(this.options.language, 'errors.eagleEye.keywordsMissing') } + // We need at least one platform if (!data.platforms || data.platforms.length === 0) { throw new Error400(this.options.language, 'errors.eagleEye.platformMissing') } + // Make sure platforms are in the allowed list const platforms = Object.values(EagleEyePlatforms) as string[] data.platforms.forEach((platform) => { if (!platforms.includes(platform)) { @@ -46,6 +55,7 @@ export default class EagleEyeSettingsService extends LoggingBase { } }) + // We need a date. Make sure it's in the allowed list. const publishedDates = Object.values(EagleEyePublishedDates) as string[] if (publishedDates.indexOf(data.publishedDate as string) === -1) { throw new Error400( @@ -55,7 +65,10 @@ export default class EagleEyeSettingsService extends LoggingBase { ) } + // Convert the relative string date to a Date data.publishedDate = EagleEyeSettingsService.switchDate(data.publishedDate as string) + + // Remove any extra fields return lodash.pick(data, [ 'keywords', 'exactKeywords', @@ -65,6 +78,11 @@ export default class EagleEyeSettingsService extends LoggingBase { ]) } + /** + * Convert a relative string date to a Date. For example, 30 days ago -> 2020-01-01 + * @param date String date. Can be one of EagleEyePublishedDates + * @returns The corresponding Date + */ static switchDate(date: string) { switch (date) { case 'Last 24h': @@ -82,48 +100,70 @@ export default class EagleEyeSettingsService extends LoggingBase { } } + /** + * Validate and normalize email digest settings. + * @param data Email digest settings of type EagleEyeEmailDigestSettings + * @param feed Standard feed settings of type EagleEyeFeedSettings + * @returns The normalized email digest settings if the input is valid. Otherwise a 400 Error. + */ getEmailDigestSettings(data: EagleEyeEmailDigestSettings, feed: EagleEyeFeedSettings) { + // If the matchFeedSettings option is toggled, we set the email feed settings to the standard feed settings. + // Otherwise, we validate and normalize the email feed settings. if (!data.matchFeedSettings) { data.feed = this.getFeed(data.feed) } else { data.feed = feed } + // Make sure the email exists and is valid const emailRegex = /^(([^<>()[\]\\.,;:\s@"]+(\.[^<>()[\]\\.,;:\s@"]+)*)|(".+"))@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\])|(([a-zA-Z\-0-9]+\.)+[a-zA-Z]{2,}))$/ if (!emailRegex.test(data.email)) { throw new Error400(this.options.language, 'errors.eagleEye.emailInvalid') } + // Make sure the frequency exists and is valid if (['daily', 'weekly'].indexOf(data.frequency) === -1) { throw new Error400(this.options.language, 'errors.eagleEye.frequencyInvalid') } + // Make sure the time exists and is valid const timeRegex = /^([0-1]?[0-9]|2[0-3]):[0-5][0-9]?$/ if (!timeRegex.test(data.time)) { throw new Error400(this.options.language, 'errors.eagleEye.timeInvalid') } + // Remove any extra fields return lodash.pick(data, ['email', 'frequency', 'time', 'matchFeedSettings', 'feed']) } + /** + * Validate, normalize and update EagleEye settings. + * @param data Input of type EagleEyeSettings + * @returns Normalized EagleEyeSettings if the input is valid. Otherwise a 400 Error. + */ async update(data: EagleEyeSettings): Promise { const transaction = await SequelizeRepository.createTransaction(this.options) try { + // At this point onboarded is true always data.onboarded = true + // Validate and normalize feed settings data.feed = this.getFeed(data.feed) + // If an email digest was sent, validate and normalize email digest settings + // Otherwise, set email digest to false if (data.emailDigest || data.emailDigestActive) { data.emailDigestActive = true - data.emailDigest = this.getEmailDigestSettings(data.emailDigest, data.feed) } else { data.emailDigestActive = false } + // Remove any extra fields data = lodash.pick(data, ['onboarded', 'feed', 'emailDigestActive', 'emailDigest']) + // Update the user's EagleEye settings const userOut = await UserRepository.updateEagleEyeSettings( this.options.currentUser.id, { eagleEyeSettings: data }, From 729828f72159e5ccd86254594378df45afdd882d Mon Sep 17 00:00:00 2001 From: Joan Reyero Date: Tue, 7 Feb 2023 15:23:44 +0100 Subject: [PATCH 30/36] Fixed frontend sync --- backend/src/services/eagleEyeSettingsService.ts | 2 +- backend/src/types/eagleEyeTypes.ts | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/backend/src/services/eagleEyeSettingsService.ts b/backend/src/services/eagleEyeSettingsService.ts index d500a8ff40..011f5879fe 100644 --- a/backend/src/services/eagleEyeSettingsService.ts +++ b/backend/src/services/eagleEyeSettingsService.ts @@ -166,7 +166,7 @@ export default class EagleEyeSettingsService extends LoggingBase { // Update the user's EagleEye settings const userOut = await UserRepository.updateEagleEyeSettings( this.options.currentUser.id, - { eagleEyeSettings: data }, + data, { ...this.options, transaction }, ) diff --git a/backend/src/types/eagleEyeTypes.ts b/backend/src/types/eagleEyeTypes.ts index 2f1f2d4d59..91bc67883f 100644 --- a/backend/src/types/eagleEyeTypes.ts +++ b/backend/src/types/eagleEyeTypes.ts @@ -65,8 +65,8 @@ export enum EagleEyePlatforms { export enum EagleEyePublishedDates { LAST_24_HOURS = 'Last 24h', - LAST_7_DAYS = 'Last 7 days', - LAST_14_DAYS = 'Last 14 days', - LAST_30_DAYS = 'Last 30 days', - LAST_90_DAYS = 'Last 90 days', + LAST_7_DAYS = 'Last 7d', + LAST_14_DAYS = 'Last 14d', + LAST_30_DAYS = 'Last 30d', + LAST_90_DAYS = 'Last 90d', } From 3e05210265de09efb78d8d0c41c677d43bec6adf Mon Sep 17 00:00:00 2001 From: Joan Reyero Date: Tue, 7 Feb 2023 17:09:28 +0100 Subject: [PATCH 31/36] Added EagleEye search --- backend/.env.dist.local | 6 ++- .../config/custom-environment-variables.json | 4 ++ .../eagleEyeContent/eagleEyeContentSearch.ts | 8 ++-- backend/src/api/eagleEyeContent/index.ts | 9 ++-- backend/src/config/configTypes.ts | 5 ++ backend/src/config/index.ts | 8 ++++ backend/src/i18n/en.ts | 1 + .../src/services/eagleEyeContentService.ts | 46 ++++++++++++++++--- .../src/services/eagleEyeSettingsService.ts | 10 ++-- 9 files changed, 75 insertions(+), 22 deletions(-) diff --git a/backend/.env.dist.local b/backend/.env.dist.local index dbec134ba0..ad0575574e 100755 --- a/backend/.env.dist.local +++ b/backend/.env.dist.local @@ -133,4 +133,8 @@ CROWD_QDRANT_PORT=6333 # Enrichment settings CROWD_ENRICHMENT_URL= -CROWD_ENRICHMENT_API_KEY= \ No newline at end of file +CROWD_ENRICHMENT_API_KEY= + +# EagleEye settings +CROWD_EAGLE_EYE_URL= +CROWD_EAGLE_EYE_API_KEY= \ No newline at end of file diff --git a/backend/config/custom-environment-variables.json b/backend/config/custom-environment-variables.json index 56b8645816..862d9e9fb9 100644 --- a/backend/config/custom-environment-variables.json +++ b/backend/config/custom-environment-variables.json @@ -135,5 +135,9 @@ "enrichment": { "url": "CROWD_ENRICHMENT_URL", "apiKey": "CROWD_ENRICHMENT_API_KEY" + }, + "eagleEye": { + "url": "CROWD_EAGLE_EYE_URL", + "apiKey": "CROWD_EAGLE_EYE_API_KEY" } } diff --git a/backend/src/api/eagleEyeContent/eagleEyeContentSearch.ts b/backend/src/api/eagleEyeContent/eagleEyeContentSearch.ts index 278459e1c3..88659e218a 100644 --- a/backend/src/api/eagleEyeContent/eagleEyeContentSearch.ts +++ b/backend/src/api/eagleEyeContent/eagleEyeContentSearch.ts @@ -4,11 +4,9 @@ import EagleEyeContentService from '../../services/eagleEyeContentService' import PermissionChecker from '../../services/user/permissionChecker' export default async (req, res) => { - new PermissionChecker(req).validateHas(Permissions.values.eagleEyeContentSearch) - - const payload = await new EagleEyeContentService(req).search(req.body) - - track('EagleEyeSearch', { ...req.body }, { ...req }) + new PermissionChecker(req).validateHas(Permissions.values.eagleEyeActionCreate) + const payload = await new EagleEyeContentService(req).search() + track('EagleEye backend search', { ...req.body }, { ...req }) await req.responseHandler.success(req, res, payload) } diff --git a/backend/src/api/eagleEyeContent/index.ts b/backend/src/api/eagleEyeContent/index.ts index cbc3f21db4..9d1192d7f3 100644 --- a/backend/src/api/eagleEyeContent/index.ts +++ b/backend/src/api/eagleEyeContent/index.ts @@ -1,15 +1,16 @@ import { safeWrap } from '../../middlewares/errorMiddleware' export default (app) => { - app.post( - `/tenant/:tenantId/eagleEyeContent`, - safeWrap(require('./eagleEyeContentSearch').default), - ) app.post( `/tenant/:tenantId/eagleEyeContent/query`, safeWrap(require('./eagleEyeContentQuery').default), ) + app.get( + `/tenant/:tenantId/eagleEyeContent/search`, + safeWrap(require('./eagleEyeContentSearch').default), + ) + app.get( `/tenant/:tenantId/eagleEyeContent/:id`, safeWrap(require('./eagleEyeContentFind').default), diff --git a/backend/src/config/configTypes.ts b/backend/src/config/configTypes.ts index c0949978df..f6ec20d69b 100644 --- a/backend/src/config/configTypes.ts +++ b/backend/src/config/configTypes.ts @@ -179,3 +179,8 @@ export interface EnrichmentConfiguration { url: string apiKey: string } + +export interface EagleEyeConfiguration { + url: string + apiKey: string +} diff --git a/backend/src/config/index.ts b/backend/src/config/index.ts index 6d32e96932..69ad9c6f72 100644 --- a/backend/src/config/index.ts +++ b/backend/src/config/index.ts @@ -24,6 +24,7 @@ import { PosthogConfiguration, PizzlyConfiguration, EnrichmentConfiguration, + EagleEyeConfiguration, } from './configTypes' // TODO-kube @@ -233,3 +234,10 @@ export const ENRICHMENT_CONFIG: EnrichmentConfiguration = KUBE_MODE url: process.env.ENRICHMENT_URL, apiKey: process.env.ENRICHMENT_SECRET_KEY, } + +export const EAGLE_EYE_CONFIG: EagleEyeConfiguration = KUBE_MODE + ? config.get('eagleEye') + : { + url: process.env.EAGLE_EYE_URL, + apiKey: process.env.EAGLE_EYE_SECRET_KEY, + } diff --git a/backend/src/i18n/en.ts b/backend/src/i18n/en.ts index f0fad60032..866d968b49 100644 --- a/backend/src/i18n/en.ts +++ b/backend/src/i18n/en.ts @@ -123,6 +123,7 @@ const en = { frequencyInvalid: 'emailDigest.frequency needs to be one of daily, weekly. Settings not updated.', timeInvalid: 'emailDigest.time needs to be a valid time. Settings not updated.', + notOnboarded: 'Eagle eye is not set up yet.', }, integrations: { badEndpoint: 'Bad endpoint: {0}', diff --git a/backend/src/services/eagleEyeContentService.ts b/backend/src/services/eagleEyeContentService.ts index 4a58b4f766..94484b1afc 100644 --- a/backend/src/services/eagleEyeContentService.ts +++ b/backend/src/services/eagleEyeContentService.ts @@ -1,9 +1,12 @@ +import axios from 'axios' +import { EAGLE_EYE_CONFIG } from '../config' import { IServiceOptions } from './IServiceOptions' import EagleEyeContentRepository from '../database/repositories/eagleEyeContentRepository' import { LoggingBase } from './loggingBase' -import { EagleEyeContent, EagleEyeAction } from '../types/eagleEyeTypes' +import { EagleEyeContent, EagleEyeAction, EagleEyeSettings } from '../types/eagleEyeTypes' import { PageData, QueryData } from '../types/common' import Error400 from '../errors/Error400' +import UserRepository from '../database/repositories/userRepository' export interface EagleEyeContentUpsertData extends EagleEyeAction { content: EagleEyeContent @@ -57,11 +60,40 @@ export default class EagleEyeContentService extends LoggingBase { ) } - /** - TODO - */ - /* eslint-disable-next-line */ - async search(args) { - return null + async search(email = false) { + const eagleEyeSettings: EagleEyeSettings = ( + await UserRepository.findById(this.options.currentUser.id, this.options) + ).eagleEyeSettings + + if (!eagleEyeSettings.onboarded) { + throw new Error400(this.options.language, 'errors.eagleEye.notOnboarded') + } + + const feedSettings = email ? eagleEyeSettings.emailDigest.feed : eagleEyeSettings.feed + + const keywords = feedSettings.keywords ? feedSettings.keywords.join(',') : '' + const exactKeywords = feedSettings.exactKeywords ? feedSettings.exactKeywords.join(',') : '' + const excludedKeywords = feedSettings.excludedKeywords + ? feedSettings.excludedKeywords.join(',') + : '' + + const config = { + method: 'get', + maxBodyLength: Infinity, + url: `${EAGLE_EYE_CONFIG.url}`, + params: { + platforms: feedSettings.platforms.join(','), + keywords, + exact_keywords: exactKeywords, + exclude_keywords: excludedKeywords, + after_date: feedSettings.publishedDate, + }, + headers: { + Authorization: `Bearer ${EAGLE_EYE_CONFIG.apiKey}`, + }, + } + + const response = await axios(config) + return response.data } } diff --git a/backend/src/services/eagleEyeSettingsService.ts b/backend/src/services/eagleEyeSettingsService.ts index 011f5879fe..fc422a4784 100644 --- a/backend/src/services/eagleEyeSettingsService.ts +++ b/backend/src/services/eagleEyeSettingsService.ts @@ -85,15 +85,15 @@ export default class EagleEyeSettingsService extends LoggingBase { */ static switchDate(date: string) { switch (date) { - case 'Last 24h': + case EagleEyePublishedDates.LAST_24_HOURS: return moment().subtract(1, 'days').format('YYYY-MM-DD') - case 'Last 7 days': + case EagleEyePublishedDates.LAST_7_DAYS: return moment().subtract(7, 'days').format('YYYY-MM-DD') - case 'Last 14 days': + case EagleEyePublishedDates.LAST_14_DAYS: return moment().subtract(14, 'days').format('YYYY-MM-DD') - case 'Last 30 days': + case EagleEyePublishedDates.LAST_30_DAYS: return moment().subtract(30, 'days').format('YYYY-MM-DD') - case 'Last 90 days': + case EagleEyePublishedDates.LAST_90_DAYS: return moment().subtract(90, 'days').format('YYYY-MM-DD') default: return null From b3d4eb073c33056c27946fea586a812e84027e71 Mon Sep 17 00:00:00 2001 From: Joan Reyero Date: Tue, 7 Feb 2023 17:28:59 +0100 Subject: [PATCH 32/36] Added fixes and missing endpoints --- backend/src/api/eagleEyeContent/index.ts | 5 +++ .../src/services/eagleEyeContentService.ts | 36 +++++++++++++++++-- .../src/services/eagleEyeSettingsService.ts | 28 +-------------- backend/src/types/eagleEyeTypes.ts | 2 +- 4 files changed, 41 insertions(+), 30 deletions(-) diff --git a/backend/src/api/eagleEyeContent/index.ts b/backend/src/api/eagleEyeContent/index.ts index 9d1192d7f3..776e6ce43f 100644 --- a/backend/src/api/eagleEyeContent/index.ts +++ b/backend/src/api/eagleEyeContent/index.ts @@ -6,6 +6,11 @@ export default (app) => { safeWrap(require('./eagleEyeContentQuery').default), ) + app.post( + `/tenant/:tenantId/eagleEyeContent`, + safeWrap(require('./eagleEyeContentUpsert').default), + ) + app.get( `/tenant/:tenantId/eagleEyeContent/search`, safeWrap(require('./eagleEyeContentSearch').default), diff --git a/backend/src/services/eagleEyeContentService.ts b/backend/src/services/eagleEyeContentService.ts index 94484b1afc..e5cfefd057 100644 --- a/backend/src/services/eagleEyeContentService.ts +++ b/backend/src/services/eagleEyeContentService.ts @@ -1,9 +1,15 @@ +import moment from 'moment' import axios from 'axios' import { EAGLE_EYE_CONFIG } from '../config' import { IServiceOptions } from './IServiceOptions' import EagleEyeContentRepository from '../database/repositories/eagleEyeContentRepository' import { LoggingBase } from './loggingBase' -import { EagleEyeContent, EagleEyeAction, EagleEyeSettings } from '../types/eagleEyeTypes' +import { + EagleEyeContent, + EagleEyeAction, + EagleEyeSettings, + EagleEyePublishedDates, +} from '../types/eagleEyeTypes' import { PageData, QueryData } from '../types/common' import Error400 from '../errors/Error400' import UserRepository from '../database/repositories/userRepository' @@ -60,6 +66,28 @@ export default class EagleEyeContentService extends LoggingBase { ) } + /** + * Convert a relative string date to a Date. For example, 30 days ago -> 2020-01-01 + * @param date String date. Can be one of EagleEyePublishedDates + * @returns The corresponding Date + */ + static switchDate(date: string) { + switch (date) { + case EagleEyePublishedDates.LAST_24_HOURS: + return moment().subtract(1, 'days').format('YYYY-MM-DD') + case EagleEyePublishedDates.LAST_7_DAYS: + return moment().subtract(7, 'days').format('YYYY-MM-DD') + case EagleEyePublishedDates.LAST_14_DAYS: + return moment().subtract(14, 'days').format('YYYY-MM-DD') + case EagleEyePublishedDates.LAST_30_DAYS: + return moment().subtract(30, 'days').format('YYYY-MM-DD') + case EagleEyePublishedDates.LAST_90_DAYS: + return moment().subtract(90, 'days').format('YYYY-MM-DD') + default: + return null + } + } + async search(email = false) { const eagleEyeSettings: EagleEyeSettings = ( await UserRepository.findById(this.options.currentUser.id, this.options) @@ -77,6 +105,8 @@ export default class EagleEyeContentService extends LoggingBase { ? feedSettings.excludedKeywords.join(',') : '' + const afterDate = EagleEyeContentService.switchDate(feedSettings.publishedDate) + const config = { method: 'get', maxBodyLength: Infinity, @@ -86,7 +116,7 @@ export default class EagleEyeContentService extends LoggingBase { keywords, exact_keywords: exactKeywords, exclude_keywords: excludedKeywords, - after_date: feedSettings.publishedDate, + after_date: afterDate, }, headers: { Authorization: `Bearer ${EAGLE_EYE_CONFIG.apiKey}`, @@ -94,6 +124,8 @@ export default class EagleEyeContentService extends LoggingBase { } const response = await axios(config) + + // const interacted = this.query({ filter: { user: this.options.currentUser.id } }) return response.data } } diff --git a/backend/src/services/eagleEyeSettingsService.ts b/backend/src/services/eagleEyeSettingsService.ts index fc422a4784..7753f2f13d 100644 --- a/backend/src/services/eagleEyeSettingsService.ts +++ b/backend/src/services/eagleEyeSettingsService.ts @@ -1,4 +1,3 @@ -import moment from 'moment' import lodash from 'lodash' import SequelizeRepository from '../database/repositories/sequelizeRepository' import UserRepository from '../database/repositories/userRepository' @@ -57,7 +56,7 @@ export default class EagleEyeSettingsService extends LoggingBase { // We need a date. Make sure it's in the allowed list. const publishedDates = Object.values(EagleEyePublishedDates) as string[] - if (publishedDates.indexOf(data.publishedDate as string) === -1) { + if (publishedDates.indexOf(data.publishedDate) === -1) { throw new Error400( this.options.language, 'errors.eagleEye.publishedDateMissing', @@ -65,9 +64,6 @@ export default class EagleEyeSettingsService extends LoggingBase { ) } - // Convert the relative string date to a Date - data.publishedDate = EagleEyeSettingsService.switchDate(data.publishedDate as string) - // Remove any extra fields return lodash.pick(data, [ 'keywords', @@ -78,28 +74,6 @@ export default class EagleEyeSettingsService extends LoggingBase { ]) } - /** - * Convert a relative string date to a Date. For example, 30 days ago -> 2020-01-01 - * @param date String date. Can be one of EagleEyePublishedDates - * @returns The corresponding Date - */ - static switchDate(date: string) { - switch (date) { - case EagleEyePublishedDates.LAST_24_HOURS: - return moment().subtract(1, 'days').format('YYYY-MM-DD') - case EagleEyePublishedDates.LAST_7_DAYS: - return moment().subtract(7, 'days').format('YYYY-MM-DD') - case EagleEyePublishedDates.LAST_14_DAYS: - return moment().subtract(14, 'days').format('YYYY-MM-DD') - case EagleEyePublishedDates.LAST_30_DAYS: - return moment().subtract(30, 'days').format('YYYY-MM-DD') - case EagleEyePublishedDates.LAST_90_DAYS: - return moment().subtract(90, 'days').format('YYYY-MM-DD') - default: - return null - } - } - /** * Validate and normalize email digest settings. * @param data Email digest settings of type EagleEyeEmailDigestSettings diff --git a/backend/src/types/eagleEyeTypes.ts b/backend/src/types/eagleEyeTypes.ts index 91bc67883f..f085d6645c 100644 --- a/backend/src/types/eagleEyeTypes.ts +++ b/backend/src/types/eagleEyeTypes.ts @@ -28,7 +28,7 @@ export interface EagleEyeFeedSettings { keywords: string[] exactKeywords: string[] excludedKeywords: string[] - publishedDate: string | Date + publishedDate: string platforms: string[] } From bb6582d4cb4dcbdaad670e1ddad04760c59708f1 Mon Sep 17 00:00:00 2001 From: Joan Reyero Date: Tue, 7 Feb 2023 18:37:06 +0100 Subject: [PATCH 33/36] Added proper search --- .../src/services/eagleEyeContentService.ts | 54 ++++++++++++++++--- backend/src/types/eagleEyeTypes.ts | 21 ++++++++ 2 files changed, 67 insertions(+), 8 deletions(-) diff --git a/backend/src/services/eagleEyeContentService.ts b/backend/src/services/eagleEyeContentService.ts index e5cfefd057..3220cef2f1 100644 --- a/backend/src/services/eagleEyeContentService.ts +++ b/backend/src/services/eagleEyeContentService.ts @@ -9,6 +9,8 @@ import { EagleEyeAction, EagleEyeSettings, EagleEyePublishedDates, + EagleEyeRawPost, + EagleEyePostWithActions, } from '../types/eagleEyeTypes' import { PageData, QueryData } from '../types/common' import Error400 from '../errors/Error400' @@ -71,21 +73,28 @@ export default class EagleEyeContentService extends LoggingBase { * @param date String date. Can be one of EagleEyePublishedDates * @returns The corresponding Date */ - static switchDate(date: string) { + static switchDate(date: string, offset = 0) { + let dateMoment switch (date) { case EagleEyePublishedDates.LAST_24_HOURS: - return moment().subtract(1, 'days').format('YYYY-MM-DD') + dateMoment = moment().subtract(1, 'days') + break case EagleEyePublishedDates.LAST_7_DAYS: - return moment().subtract(7, 'days').format('YYYY-MM-DD') + dateMoment = moment().subtract(7, 'days') + break case EagleEyePublishedDates.LAST_14_DAYS: - return moment().subtract(14, 'days').format('YYYY-MM-DD') + dateMoment = moment().subtract(14, 'days') + break case EagleEyePublishedDates.LAST_30_DAYS: - return moment().subtract(30, 'days').format('YYYY-MM-DD') + dateMoment = moment().subtract(30, 'days') + break case EagleEyePublishedDates.LAST_90_DAYS: - return moment().subtract(90, 'days').format('YYYY-MM-DD') + dateMoment = moment().subtract(90, 'days') + break default: return null } + return dateMoment.add(offset, 'days').format('YYYY-MM-DD') } async search(email = false) { @@ -125,7 +134,36 @@ export default class EagleEyeContentService extends LoggingBase { const response = await axios(config) - // const interacted = this.query({ filter: { user: this.options.currentUser.id } }) - return response.data + const interacted = ( + await this.query({ + filter: { + postedAt: { gt: EagleEyeContentService.switchDate(feedSettings.publishedDate, 15) }, + }, + }) + ).rows + + const interactedMap = {} + + for (const item of interacted) { + interactedMap[item.url] = item + } + + const out: EagleEyePostWithActions[] = [] + for (const item of response.data as EagleEyeRawPost[]) { + const post = { + description: item.description, + thumbnail: item.thumbnail, + title: item.title, + } + out.push({ + url: item.url, + postedAt: item.date, + post, + platform: item.platform, + actions: interactedMap[item.url] ? interactedMap[item.url].actions : [], + }) + } + + return out } } diff --git a/backend/src/types/eagleEyeTypes.ts b/backend/src/types/eagleEyeTypes.ts index f085d6645c..2dccce7703 100644 --- a/backend/src/types/eagleEyeTypes.ts +++ b/backend/src/types/eagleEyeTypes.ts @@ -70,3 +70,24 @@ export enum EagleEyePublishedDates { LAST_30_DAYS = 'Last 30d', LAST_90_DAYS = 'Last 90d', } + +export interface EagleEyeRawPost { + description: string + title: string + thumbnail?: string + url: string + platform: string + date: string +} + +export interface EagleEyePostWithActions { + post: { + description: string + title: string + thumbnail?: string + } + url: string + platform: string + postedAt: string + actions: EagleEyeAction[] +} From 14856113c14d2403b95fe282096c12e780af24e6 Mon Sep 17 00:00:00 2001 From: Joan Reyero Date: Wed, 8 Feb 2023 11:20:01 +0100 Subject: [PATCH 34/36] Is feature flagging done? --- backend/src/api/eagleEyeContent/index.ts | 3 +++ backend/src/i18n/en.ts | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/backend/src/api/eagleEyeContent/index.ts b/backend/src/api/eagleEyeContent/index.ts index 776e6ce43f..15eb591d57 100644 --- a/backend/src/api/eagleEyeContent/index.ts +++ b/backend/src/api/eagleEyeContent/index.ts @@ -1,4 +1,6 @@ import { safeWrap } from '../../middlewares/errorMiddleware' +import { featureFlagMiddleware } from '../../middlewares/featureFlagMiddleware' +import { FeatureFlag } from '../../types/common' export default (app) => { app.post( @@ -13,6 +15,7 @@ export default (app) => { app.get( `/tenant/:tenantId/eagleEyeContent/search`, + featureFlagMiddleware(FeatureFlag.EAGLE_EYE, 'entities.eagleEye.errors.planLimitExceeded'), safeWrap(require('./eagleEyeContentSearch').default), ) diff --git a/backend/src/i18n/en.ts b/backend/src/i18n/en.ts index 866d968b49..7df609a98f 100644 --- a/backend/src/i18n/en.ts +++ b/backend/src/i18n/en.ts @@ -205,6 +205,12 @@ const en = { planLimitExceeded: 'You have exceeded # of automations you can have in your plan.', }, }, + eagleEye: { + errors: { + planLimitExceeded: + 'EagleEye is only available in the Growth and Custom plans. Please upgrade your plan.', + }, + }, }, communityHelpCenter: { From be317b20f70061ba71e89f86598f68046209f497 Mon Sep 17 00:00:00 2001 From: Joan Reyero Date: Wed, 8 Feb 2023 11:25:45 +0100 Subject: [PATCH 35/36] Protected all EagleEye endpoints --- backend/src/api/eagleEyeContent/index.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/backend/src/api/eagleEyeContent/index.ts b/backend/src/api/eagleEyeContent/index.ts index 15eb591d57..7fd10c716c 100644 --- a/backend/src/api/eagleEyeContent/index.ts +++ b/backend/src/api/eagleEyeContent/index.ts @@ -5,11 +5,13 @@ import { FeatureFlag } from '../../types/common' export default (app) => { app.post( `/tenant/:tenantId/eagleEyeContent/query`, + featureFlagMiddleware(FeatureFlag.EAGLE_EYE, 'entities.eagleEye.errors.planLimitExceeded'), safeWrap(require('./eagleEyeContentQuery').default), ) app.post( `/tenant/:tenantId/eagleEyeContent`, + featureFlagMiddleware(FeatureFlag.EAGLE_EYE, 'entities.eagleEye.errors.planLimitExceeded'), safeWrap(require('./eagleEyeContentUpsert').default), ) @@ -21,21 +23,25 @@ export default (app) => { app.get( `/tenant/:tenantId/eagleEyeContent/:id`, + featureFlagMiddleware(FeatureFlag.EAGLE_EYE, 'entities.eagleEye.errors.planLimitExceeded'), safeWrap(require('./eagleEyeContentFind').default), ) app.post( `/tenant/:tenantId/eagleEyeContent/:contentId/action`, + featureFlagMiddleware(FeatureFlag.EAGLE_EYE, 'entities.eagleEye.errors.planLimitExceeded'), safeWrap(require('./eagleEyeActionCreate').default), ) app.put( `/tenant/:tenantId/eagleEyeContent/settings`, + featureFlagMiddleware(FeatureFlag.EAGLE_EYE, 'entities.eagleEye.errors.planLimitExceeded'), safeWrap(require('./eagleEyeSettingsUpdate').default), ) app.delete( `/tenant/:tenantId/eagleEyeContent/:contentId/action/:actionId`, + featureFlagMiddleware(FeatureFlag.EAGLE_EYE, 'entities.eagleEye.errors.planLimitExceeded'), safeWrap(require('./eagleEyeActionDestroy').default), ) } From 9a94a1465463e6b522b8281c62d28d0fe707955d Mon Sep 17 00:00:00 2001 From: Joan Reyero Date: Wed, 8 Feb 2023 12:58:54 +0100 Subject: [PATCH 36/36] Re-made hackernews integration (#494) --- backend/config/default.json | 3 +- .../hackerNewsIntegrationService.ts | 12 +++-- .../integrations/types/hackerNewsTypes.ts | 53 ++++++++++++------- .../usecases/hackerNews/getPostsByKeywords.ts | 47 ++++++++++++---- 4 files changed, 81 insertions(+), 34 deletions(-) diff --git a/backend/config/default.json b/backend/config/default.json index c31da9bcbb..13dfcfeaed 100644 --- a/backend/config/default.json +++ b/backend/config/default.json @@ -34,5 +34,6 @@ "maxRetrospectInSeconds": 3600 }, "github": {}, - "enrichment": {} + "enrichment": {}, + "eagleEye": {} } diff --git a/backend/src/serverless/integrations/services/integrations/hackerNewsIntegrationService.ts b/backend/src/serverless/integrations/services/integrations/hackerNewsIntegrationService.ts index f454b6171b..254125c88e 100644 --- a/backend/src/serverless/integrations/services/integrations/hackerNewsIntegrationService.ts +++ b/backend/src/serverless/integrations/services/integrations/hackerNewsIntegrationService.ts @@ -1,4 +1,5 @@ import sanitizeHtml from 'sanitize-html' +import moment from 'moment' import { MemberAttributeName } from '../../../../database/attributes/member/enums' import { HackerNewsMemberAttributes } from '../../../../database/attributes/member/hackerNews' import MemberAttributeSettingsService from '../../../../services/memberAttributeSettingsService' @@ -13,9 +14,9 @@ import Operations from '../../../dbOperations/operations' import getPost from '../../usecases/hackerNews/getPost' import { HackerNewsGrid } from '../../grid/hackerNewsGrid' import { - EagleEyeResponse, HackerNewsResponse, HackerNewsIntegrationSettings, + HackerNewsSearchResult, } from '../../types/hackerNewsTypes' import { AddActivitiesSingle } from '../../types/messageTypes' import getPostsByKeywords from '../../usecases/hackerNews/getPostsByKeywords' @@ -43,7 +44,10 @@ export class HackerNewsIntegrationService extends IntegrationServiceBase { const keywords = Array.from(new Set([...settings.keywords, ...settings.urls])) this.logger(context).info(`Fetching posts for keywords: ${keywords}`) const posts = await getPostsByKeywords( - { keywords, nDays: context.onboarding ? 1000000 : 3 }, + { + keywords, + after: context.onboarding ? 0 : moment().subtract(30, 'days').unix(), + }, context.serviceContext, this.logger(context), ) @@ -55,8 +59,8 @@ export class HackerNewsIntegrationService extends IntegrationServiceBase { } async getStreams(context: IStepContext): Promise { - return context.pipelineData.posts.map((a: EagleEyeResponse) => ({ - value: a.sourceId.slice(a.sourceId.lastIndexOf(':') + 1), + return context.pipelineData.posts.map((a: HackerNewsSearchResult) => ({ + value: a.postId, metadata: { channel: a.keywords[0], }, diff --git a/backend/src/serverless/integrations/types/hackerNewsTypes.ts b/backend/src/serverless/integrations/types/hackerNewsTypes.ts index 89bc808f3a..5093679316 100644 --- a/backend/src/serverless/integrations/types/hackerNewsTypes.ts +++ b/backend/src/serverless/integrations/types/hackerNewsTypes.ts @@ -1,22 +1,37 @@ -export interface EagleEyeResponse { - vectorId: number - sourceId: string - title: string - url: string - createdAt: string - text: string - username: string - platform: string - timestamp: string - userAttributes: any - postAttributes: { - commentsCount: number - score: number - } - keywords: string[] +export interface HackerNewsSearchResponseRaw { + hits: [ + { + created_at: string + title: string + url: string + author: string + points: number + story_text: string + comment_text: string | null + num_comments: number + story_id: string | null + story_title: string | null + story_url: string | null + parent_id: string | null + created_at_i: number + _tags: string[] + objectID: string + _highlightResult: { + [key: string]: { + value: string + matchLevel: string + matchedWords: string[] + fullyHighlighted?: boolean + } + } + }, + ] } -export type EagleEyeResponses = EagleEyeResponse[] +export interface HackerNewsSearchResult { + keywords: string[] + postId: number +} export interface HackerNewsPost { by: string @@ -49,7 +64,7 @@ export interface HackerNewsIntegrationSettings { urls: string[] } -export interface EagleEyeInput { +export interface HackerNewsKeywordSearchInput { keywords: string[] - nDays: number + after: number } diff --git a/backend/src/serverless/integrations/usecases/hackerNews/getPostsByKeywords.ts b/backend/src/serverless/integrations/usecases/hackerNews/getPostsByKeywords.ts index 0e19a3ac5e..6a3563b131 100644 --- a/backend/src/serverless/integrations/usecases/hackerNews/getPostsByKeywords.ts +++ b/backend/src/serverless/integrations/usecases/hackerNews/getPostsByKeywords.ts @@ -1,23 +1,50 @@ +import axios from 'axios' import { IServiceOptions } from '../../../../services/IServiceOptions' -import { EagleEyeResponses, EagleEyeInput } from '../../types/hackerNewsTypes' +import { + HackerNewsKeywordSearchInput, + HackerNewsSearchResponseRaw, + HackerNewsSearchResult, +} from '../../types/hackerNewsTypes' import { Logger } from '../../../../utils/logging' import { timeout } from '../../../../utils/timing' -import EagleEyeContentService from '../../../../services/eagleEyeContentService' async function getPostsByKeyword( - input: EagleEyeInput, + input: HackerNewsKeywordSearchInput, options: IServiceOptions, logger: Logger, -): Promise { +): Promise { await timeout(2000) try { - const eagleEyeService = new EagleEyeContentService(options) - return await eagleEyeService.keywordMatch({ - keywords: input.keywords, - nDays: input.nDays, - platform: 'hacker_news', - }) + const out = [] + const existing = new Set() + for (const keyword of input.keywords) { + const config = { + method: 'get', + maxBodyLength: Infinity, + url: 'http://hn.algolia.com/api/v1/search', + params: { + query: keyword, + numericFilters: `created_at_i>${input.after}`, + tags: '(story,ask_hn,show_hn,poll)', + }, + headers: {}, + } + + const response = await axios(config) + const data = response.data as HackerNewsSearchResponseRaw + + for (const item of data.hits) { + if (!existing.has(item.objectID)) { + out.push({ + keywords: [keyword], + postId: item.objectID, + }) + existing.add(item.objectID) + } + } + } + return out } catch (err) { logger.error({ err, input }, 'Error while getting posts by keyword in EagleEye') throw err