diff --git a/backend/.env.dist.local b/backend/.env.dist.local index dbec134ba0..ad0575574e 100755 --- a/backend/.env.dist.local +++ b/backend/.env.dist.local @@ -133,4 +133,8 @@ CROWD_QDRANT_PORT=6333 # Enrichment settings CROWD_ENRICHMENT_URL= -CROWD_ENRICHMENT_API_KEY= \ No newline at end of file +CROWD_ENRICHMENT_API_KEY= + +# EagleEye settings +CROWD_EAGLE_EYE_URL= +CROWD_EAGLE_EYE_API_KEY= \ No newline at end of file diff --git a/backend/config/custom-environment-variables.json b/backend/config/custom-environment-variables.json index 56b8645816..862d9e9fb9 100644 --- a/backend/config/custom-environment-variables.json +++ b/backend/config/custom-environment-variables.json @@ -135,5 +135,9 @@ "enrichment": { "url": "CROWD_ENRICHMENT_URL", "apiKey": "CROWD_ENRICHMENT_API_KEY" + }, + "eagleEye": { + "url": "CROWD_EAGLE_EYE_URL", + "apiKey": "CROWD_EAGLE_EYE_API_KEY" } } diff --git a/backend/config/default.json b/backend/config/default.json index c31da9bcbb..13dfcfeaed 100644 --- a/backend/config/default.json +++ b/backend/config/default.json @@ -34,5 +34,6 @@ "maxRetrospectInSeconds": 3600 }, "github": {}, - "enrichment": {} + "enrichment": {}, + "eagleEye": {} } diff --git a/backend/src/api/eagleEyeContent/eagleEyeActionCreate.ts b/backend/src/api/eagleEyeContent/eagleEyeActionCreate.ts new file mode 100644 index 0000000000..8d25a39212 --- /dev/null +++ b/backend/src/api/eagleEyeContent/eagleEyeActionCreate.ts @@ -0,0 +1,11 @@ +import Permissions from '../../security/permissions' +import EagleEyeActionService from '../../services/eagleEyeActionService' +import PermissionChecker from '../../services/user/permissionChecker' + +export default async (req, res) => { + new PermissionChecker(req).validateHas(Permissions.values.eagleEyeActionCreate) + + const payload = await new EagleEyeActionService(req).create(req.body, req.params.contentId) + + await req.responseHandler.success(req, res, payload) +} diff --git a/backend/src/api/eagleEyeContent/eagleEyeActionDestroy.ts b/backend/src/api/eagleEyeContent/eagleEyeActionDestroy.ts new file mode 100644 index 0000000000..7684b74174 --- /dev/null +++ b/backend/src/api/eagleEyeContent/eagleEyeActionDestroy.ts @@ -0,0 +1,11 @@ +import Permissions from '../../security/permissions' +import EagleEyeActionService from '../../services/eagleEyeActionService' +import PermissionChecker from '../../services/user/permissionChecker' + +export default async (req, res) => { + new PermissionChecker(req).validateHas(Permissions.values.eagleEyeActionDestroy) + + const payload = await new EagleEyeActionService(req).destroy(req.params.actionId) + + await req.responseHandler.success(req, res, payload) +} diff --git a/backend/src/api/eagleEyeContent/eagleEyeContentList.ts b/backend/src/api/eagleEyeContent/eagleEyeContentList.ts deleted file mode 100644 index 40fd0fb9c9..0000000000 --- a/backend/src/api/eagleEyeContent/eagleEyeContentList.ts +++ /dev/null @@ -1,17 +0,0 @@ -import Permissions from '../../security/permissions' -import track from '../../segment/track' -import EagleEyeContentService from '../../services/eagleEyeContentService' -import PermissionChecker from '../../services/user/permissionChecker' - -export default async (req, res) => { - new PermissionChecker(req).validateHas(Permissions.values.eagleEyeContentRead) - const payload = await new EagleEyeContentService(req).findAndCountAll(req.query) - - if (req.query.filter && Object.keys(req.query.filter).length > 0) { - const platforms = req.query.filter.platforms ? req.query.filter.platforms.split(',') : [] - const nDays = req.query.filter.nDays - track('Eagle Eye Filter', { filter: req.query.filter, platforms, nDays }, { ...req }) - } - - await req.responseHandler.success(req, res, payload) -} diff --git a/backend/src/api/eagleEyeContent/eagleEyeContentSearch.ts b/backend/src/api/eagleEyeContent/eagleEyeContentSearch.ts index 278459e1c3..88659e218a 100644 --- a/backend/src/api/eagleEyeContent/eagleEyeContentSearch.ts +++ b/backend/src/api/eagleEyeContent/eagleEyeContentSearch.ts @@ -4,11 +4,9 @@ import EagleEyeContentService from '../../services/eagleEyeContentService' import PermissionChecker from '../../services/user/permissionChecker' export default async (req, res) => { - new PermissionChecker(req).validateHas(Permissions.values.eagleEyeContentSearch) - - const payload = await new EagleEyeContentService(req).search(req.body) - - track('EagleEyeSearch', { ...req.body }, { ...req }) + new PermissionChecker(req).validateHas(Permissions.values.eagleEyeActionCreate) + const payload = await new EagleEyeContentService(req).search() + track('EagleEye backend search', { ...req.body }, { ...req }) await req.responseHandler.success(req, res, payload) } diff --git a/backend/src/api/eagleEyeContent/eagleEyeContentUpdate.ts b/backend/src/api/eagleEyeContent/eagleEyeContentUpsert.ts similarity index 77% rename from backend/src/api/eagleEyeContent/eagleEyeContentUpdate.ts rename to backend/src/api/eagleEyeContent/eagleEyeContentUpsert.ts index b1d3c0854e..2acfcc5e2d 100644 --- a/backend/src/api/eagleEyeContent/eagleEyeContentUpdate.ts +++ b/backend/src/api/eagleEyeContent/eagleEyeContentUpsert.ts @@ -3,9 +3,9 @@ import EagleEyeContentService from '../../services/eagleEyeContentService' import PermissionChecker from '../../services/user/permissionChecker' export default async (req, res) => { - new PermissionChecker(req).validateHas(Permissions.values.eagleEyeContentEdit) + new PermissionChecker(req).validateHas(Permissions.values.eagleEyeContentCreate) - const payload = await new EagleEyeContentService(req).update(req.params.id, req.body) + const payload = await new EagleEyeContentService(req).upsert(req.body) await req.responseHandler.success(req, res, payload) } diff --git a/backend/src/api/eagleEyeContent/eagleEyeSettingsUpdate.ts b/backend/src/api/eagleEyeContent/eagleEyeSettingsUpdate.ts new file mode 100644 index 0000000000..635e4a7411 --- /dev/null +++ b/backend/src/api/eagleEyeContent/eagleEyeSettingsUpdate.ts @@ -0,0 +1,11 @@ +import Permissions from '../../security/permissions' +import EagleEyeSettingsService from '../../services/eagleEyeSettingsService' +import PermissionChecker from '../../services/user/permissionChecker' + +export default async (req, res) => { + new PermissionChecker(req).validateHas(Permissions.values.eagleEyeActionCreate) + + const payload = await new EagleEyeSettingsService(req).update(req.body) + + await req.responseHandler.success(req, res, payload) +} diff --git a/backend/src/api/eagleEyeContent/index.ts b/backend/src/api/eagleEyeContent/index.ts index 33928e9148..7fd10c716c 100644 --- a/backend/src/api/eagleEyeContent/index.ts +++ b/backend/src/api/eagleEyeContent/index.ts @@ -1,21 +1,47 @@ import { safeWrap } from '../../middlewares/errorMiddleware' +import { featureFlagMiddleware } from '../../middlewares/featureFlagMiddleware' +import { FeatureFlag } from '../../types/common' export default (app) => { - app.post( - `/tenant/:tenantId/eagleEyeContent`, - safeWrap(require('./eagleEyeContentSearch').default), - ) app.post( `/tenant/:tenantId/eagleEyeContent/query`, + featureFlagMiddleware(FeatureFlag.EAGLE_EYE, 'entities.eagleEye.errors.planLimitExceeded'), safeWrap(require('./eagleEyeContentQuery').default), ) - app.put( - `/tenant/:tenantId/eagleEyeContent/:id`, - safeWrap(require('./eagleEyeContentUpdate').default), + + app.post( + `/tenant/:tenantId/eagleEyeContent`, + featureFlagMiddleware(FeatureFlag.EAGLE_EYE, 'entities.eagleEye.errors.planLimitExceeded'), + safeWrap(require('./eagleEyeContentUpsert').default), + ) + + app.get( + `/tenant/:tenantId/eagleEyeContent/search`, + featureFlagMiddleware(FeatureFlag.EAGLE_EYE, 'entities.eagleEye.errors.planLimitExceeded'), + safeWrap(require('./eagleEyeContentSearch').default), ) - app.get(`/tenant/:tenantId/eagleEyeContent`, safeWrap(require('./eagleEyeContentList').default)) + app.get( `/tenant/:tenantId/eagleEyeContent/:id`, + featureFlagMiddleware(FeatureFlag.EAGLE_EYE, 'entities.eagleEye.errors.planLimitExceeded'), safeWrap(require('./eagleEyeContentFind').default), ) + + app.post( + `/tenant/:tenantId/eagleEyeContent/:contentId/action`, + featureFlagMiddleware(FeatureFlag.EAGLE_EYE, 'entities.eagleEye.errors.planLimitExceeded'), + safeWrap(require('./eagleEyeActionCreate').default), + ) + + app.put( + `/tenant/:tenantId/eagleEyeContent/settings`, + featureFlagMiddleware(FeatureFlag.EAGLE_EYE, 'entities.eagleEye.errors.planLimitExceeded'), + safeWrap(require('./eagleEyeSettingsUpdate').default), + ) + + app.delete( + `/tenant/:tenantId/eagleEyeContent/:contentId/action/:actionId`, + featureFlagMiddleware(FeatureFlag.EAGLE_EYE, 'entities.eagleEye.errors.planLimitExceeded'), + safeWrap(require('./eagleEyeActionDestroy').default), + ) } diff --git a/backend/src/config/configTypes.ts b/backend/src/config/configTypes.ts index c0949978df..f6ec20d69b 100644 --- a/backend/src/config/configTypes.ts +++ b/backend/src/config/configTypes.ts @@ -179,3 +179,8 @@ export interface EnrichmentConfiguration { url: string apiKey: string } + +export interface EagleEyeConfiguration { + url: string + apiKey: string +} diff --git a/backend/src/config/index.ts b/backend/src/config/index.ts index 6d32e96932..69ad9c6f72 100644 --- a/backend/src/config/index.ts +++ b/backend/src/config/index.ts @@ -24,6 +24,7 @@ import { PosthogConfiguration, PizzlyConfiguration, EnrichmentConfiguration, + EagleEyeConfiguration, } from './configTypes' // TODO-kube @@ -233,3 +234,10 @@ export const ENRICHMENT_CONFIG: EnrichmentConfiguration = KUBE_MODE url: process.env.ENRICHMENT_URL, apiKey: process.env.ENRICHMENT_SECRET_KEY, } + +export const EAGLE_EYE_CONFIG: EagleEyeConfiguration = KUBE_MODE + ? config.get('eagleEye') + : { + url: process.env.EAGLE_EYE_URL, + apiKey: process.env.EAGLE_EYE_SECRET_KEY, + } diff --git a/backend/src/database/migrations/U1675259471__eagleEyeActions.sql b/backend/src/database/migrations/U1675259471__eagleEyeActions.sql new file mode 100644 index 0000000000..30b8b2c037 --- /dev/null +++ b/backend/src/database/migrations/U1675259471__eagleEyeActions.sql @@ -0,0 +1,59 @@ +DROP TABLE IF EXISTS "eagleEyeContents"; + +DROP TYPE "eagleEyeContents_actions_type"; + +create table "eagleEyeContents"; +( + id uuid not null primary key, + "sourceId" text not null, + "vectorId" text not null, + status varchar(255) default NULL::character varying, + title text not null, + username text not null, + url text not null, + text text, + timestamp timestamp with time zone not null, + platform text not null, + keywords text [], + "similarityScore" double precision, + "userAttributes" jsonb, + "postAttributes" jsonb, + "importHash" varchar(255), + "createdAt" timestamp with time zone not null, + "updatedAt" timestamp with time zone not null, + "deletedAt" timestamp with time zone, + "tenantId" uuid not null references tenants on update cascade, + "createdById" uuid references users on update cascade on delete + set null, + "updatedById" uuid references users on update cascade on delete + set null, + "exactKeywords" text [] +); + +alter table "eagleEyeContents" owner to postgres; + +create index discord on "eagleEyeContents" ("vectorId", status); + +create index members_email_tenant_id on "eagleEyeContents" (id) +where ("deletedAt" IS NULL); + +create index members_joined_at_tenant_id on "eagleEyeContents" (id) +where ("deletedAt" IS NULL); + +create index members_username on "eagleEyeContents" using gin (id); + +create index slack on "eagleEyeContents" (id); + +create index twitter on "eagleEyeContents" (id); + +create unique index eagle_eye_contents_import_hash_tenant_id on "eagleEyeContents" ("importHash", "tenantId") +where ("deletedAt" IS NULL); + +create index eagle_eye_contents_platform_tenant_id_timestamp on "eagleEyeContents" (platform, "tenantId", timestamp) +where ("deletedAt" IS NULL); + +create index eagle_eye_contents_status_tenant_id_timestamp on "eagleEyeContents" (status, "tenantId", timestamp) +where ("deletedAt" IS NULL); + +create index eagle_eye_contents_tenant_id_timestamp on "eagleEyeContents" ("tenantId", timestamp) +where ("deletedAt" IS NULL); \ No newline at end of file diff --git a/backend/src/database/migrations/U1675702339__eagleEyeSettings.sql b/backend/src/database/migrations/U1675702339__eagleEyeSettings.sql new file mode 100644 index 0000000000..e3a0f4fe1e --- /dev/null +++ b/backend/src/database/migrations/U1675702339__eagleEyeSettings.sql @@ -0,0 +1,2 @@ +ALTER TABLE public."users" +DROP COLUMN "eagleEyeSettings"; \ No newline at end of file diff --git a/backend/src/database/migrations/V1675259471__eagleEyeActions.sql b/backend/src/database/migrations/V1675259471__eagleEyeActions.sql new file mode 100644 index 0000000000..9183281bc1 --- /dev/null +++ b/backend/src/database/migrations/V1675259471__eagleEyeActions.sql @@ -0,0 +1,33 @@ +DROP TABLE IF EXISTS "eagleEyeContents"; + +CREATE TABLE public."eagleEyeContents" ( + "id" uuid NOT NULL, + "platform" text NOT NULL, + "url" text NOT NULL, + "post" jsonb NOT NULL, + "tenantId" uuid NOT NULL, + "postedAt" timestamptz NOT NULL, + "createdAt" timestamptz NOT NULL, + "updatedAt" timestamptz NOT NULL, + CONSTRAINT "eagleEyeContents_pkey" PRIMARY KEY ("id") +); + +ALTER TABLE public."eagleEyeContents" ADD CONSTRAINT "eagleEyeContents_tenantId_fkey" FOREIGN KEY ("tenantId") REFERENCES public.tenants(id) ON DELETE NO ACTION ON UPDATE NO ACTION; + +CREATE TYPE public."eagleEyeActionTypes_type" AS ENUM ('thumbs-up', 'thumbs-down', 'bookmark'); + +CREATE TABLE public."eagleEyeActions" ( + "id" uuid NOT NULL, + "type" public."eagleEyeActionTypes_type" NOT NULL, + "timestamp" timestamptz NOT NULL, + "contentId" uuid NOT NULL, + "tenantId" uuid NOT NULL, + "actionById" uuid NOT NULL, + "createdAt" timestamptz NOT NULL, + "updatedAt" timestamptz NOT NULL, + CONSTRAINT "eagleEyeActions_pkey" PRIMARY KEY ("id") +); + +ALTER TABLE public."eagleEyeActions" ADD CONSTRAINT "eagleEyeActions_tenantId_fkey" FOREIGN KEY ("tenantId") REFERENCES public.tenants(id) ON DELETE NO ACTION ON UPDATE NO ACTION; +ALTER TABLE public."eagleEyeActions" ADD CONSTRAINT "eagleEyeActions_actionBy_fkey" FOREIGN KEY ("actionById") REFERENCES public.users(id) ON DELETE NO ACTION ON UPDATE NO ACTION; +ALTER TABLE public."eagleEyeActions" ADD CONSTRAINT "eagleEyeActions_contentId_fkey" FOREIGN KEY ("contentId") REFERENCES public."eagleEyeContents"(id) ON DELETE CASCADE ON UPDATE NO ACTION; diff --git a/backend/src/database/migrations/V1675702339__eagleEyeSettings.sql b/backend/src/database/migrations/V1675702339__eagleEyeSettings.sql new file mode 100644 index 0000000000..7e8e46fbf2 --- /dev/null +++ b/backend/src/database/migrations/V1675702339__eagleEyeSettings.sql @@ -0,0 +1,2 @@ +ALTER TABLE public."users" +ADD COLUMN "eagleEyeSettings" JSONB DEFAULT '{"onboarded": false}'; diff --git a/backend/src/database/models/eagleEyeAction.ts b/backend/src/database/models/eagleEyeAction.ts new file mode 100644 index 0000000000..116b069fb2 --- /dev/null +++ b/backend/src/database/models/eagleEyeAction.ts @@ -0,0 +1,48 @@ +import { DataTypes } from 'sequelize' + +const eagleEyeActionModel = { + id: { + type: DataTypes.UUID, + defaultValue: DataTypes.UUIDV4, + primaryKey: true, + }, + type: { + type: DataTypes.TEXT, + validate: { + isIn: [['thumbs-up', 'thumbs-down', 'bookmark']], + }, + defaultValue: null, + }, + timestamp: { + type: DataTypes.DATE, + allowNull: false, + }, +} + +export default (sequelize) => { + const eagleEyeAction = sequelize.define('eagleEyeAction', eagleEyeActionModel, { + timestamps: true, + paranoid: false, + }) + + eagleEyeAction.associate = (models) => { + models.eagleEyeAction.belongsTo(models.tenant, { + as: 'tenant', + foreignKey: { + allowNull: false, + }, + }) + + models.eagleEyeAction.belongsTo(models.user, { + as: 'actionBy', + }) + + models.eagleEyeAction.belongsTo(models.eagleEyeContent, { + as: 'content', + }) + } + + return eagleEyeAction +} + +export { eagleEyeActionModel } diff --git a/backend/src/database/models/eagleEyeContent.ts b/backend/src/database/models/eagleEyeContent.ts index bfee48a92e..9b00bc070c 100644 --- a/backend/src/database/models/eagleEyeContent.ts +++ b/backend/src/database/models/eagleEyeContent.ts @@ -6,40 +6,16 @@ const eagleEyeContentModel = { defaultValue: DataTypes.UUIDV4, primaryKey: true, }, - sourceId: { - type: DataTypes.TEXT, - allowNull: false, - validate: { - notEmpty: true, - }, - }, - vectorId: { - type: DataTypes.TEXT, - allowNull: false, - validate: { - notEmpty: true, - }, - }, - status: { - type: DataTypes.STRING(255), - validate: { - isIn: [['engaged', 'rejected']], - }, - defaultValue: null, - }, - title: { + platform: { type: DataTypes.TEXT, allowNull: false, validate: { notEmpty: true, }, }, - username: { - type: DataTypes.TEXT, + post: { + type: DataTypes.JSONB, allowNull: false, - validate: { - notEmpty: true, - }, }, url: { type: DataTypes.TEXT, @@ -48,79 +24,16 @@ const eagleEyeContentModel = { notEmpty: true, }, }, - text: { - type: DataTypes.TEXT, - }, - timestamp: { + postedAt: { type: DataTypes.DATE, allowNull: false, }, - platform: { - type: DataTypes.TEXT, - allowNull: false, - validate: { - notEmpty: true, - }, - }, - keywords: { - type: DataTypes.ARRAY(DataTypes.TEXT), - default: [], - }, - exactKeywords: { - type: DataTypes.ARRAY(DataTypes.TEXT), - default: [], - }, - similarityScore: { - type: DataTypes.FLOAT, - }, - userAttributes: { - type: DataTypes.JSONB, - default: {}, - }, - postAttributes: { - type: DataTypes.JSONB, - default: {}, - }, - importHash: { - type: DataTypes.STRING(255), - allowNull: true, - validate: { - len: [0, 255], - }, - }, } export default (sequelize) => { const eagleEyeContent = sequelize.define('eagleEyeContent', eagleEyeContentModel, { - indexes: [ - { - unique: true, - fields: ['importHash', 'tenantId'], - where: { - deletedAt: null, - }, - }, - { - fields: ['platform', 'tenantId', 'timestamp'], - where: { - deletedAt: null, - }, - }, - { - fields: ['status', 'tenantId', 'timestamp'], - where: { - deletedAt: null, - }, - }, - { - fields: ['tenantId', 'timestamp'], - where: { - deletedAt: null, - }, - }, - ], timestamps: true, - paranoid: true, + paranoid: false, }) eagleEyeContent.associate = (models) => { @@ -130,13 +43,9 @@ export default (sequelize) => { allowNull: false, }, }) - - models.eagleEyeContent.belongsTo(models.user, { - as: 'createdBy', - }) - - models.eagleEyeContent.belongsTo(models.user, { - as: 'updatedBy', + models.eagleEyeContent.hasMany(models.eagleEyeAction, { + as: 'actions', + foreignKey: 'contentId', }) } diff --git a/backend/src/database/models/index.ts b/backend/src/database/models/index.ts index 9a1dcc704c..4e0b3bcc09 100644 --- a/backend/src/database/models/index.ts +++ b/backend/src/database/models/index.ts @@ -102,6 +102,7 @@ function models() { require('./conversation').default, require('./conversationSettings').default, require('./eagleEyeContent').default, + require('./eagleEyeAction').default, require('./automation').default, require('./automationExecution').default, require('./organization').default, diff --git a/backend/src/database/models/user.ts b/backend/src/database/models/user.ts index a64482691b..3e9990465e 100644 --- a/backend/src/database/models/user.ts +++ b/backend/src/database/models/user.ts @@ -102,6 +102,13 @@ export default (sequelize, DataTypes) => { len: [0, 255], }, }, + eagleEyeSettings: { + type: DataTypes.JSONB, + allowNull: false, + defaultValue: { + onboarded: false, + }, + }, }, { indexes: [ diff --git a/backend/src/database/repositories/__tests__/eagleEyeActionRepository.test.ts b/backend/src/database/repositories/__tests__/eagleEyeActionRepository.test.ts new file mode 100644 index 0000000000..f3a64fa6ec --- /dev/null +++ b/backend/src/database/repositories/__tests__/eagleEyeActionRepository.test.ts @@ -0,0 +1,63 @@ +import EagleEyeContentRepository from '../eagleEyeContentRepository' +import SequelizeTestUtils from '../../utils/sequelizeTestUtils' +import { EagleEyeAction, EagleEyeActionType, EagleEyeContent } from '../../../types/eagleEyeTypes' +import EagleEyeActionRepository from '../eagleEyeActionRepository' + +const db = null + +describe('eagleEyeActionRepository tests', () => { + beforeEach(async () => { + await SequelizeTestUtils.wipeDatabase(db) + }) + + afterAll((done) => { + // Closing the DB connection allows Jest to exit successfully. + SequelizeTestUtils.closeConnection(db) + done() + }) + + describe('createActionForContent method', () => { + it('Should create a an action for a content succesfully', async () => { + const mockIRepositoryOptions = await SequelizeTestUtils.getTestIRepositoryOptions(db) + + const content = { + platform: 'reddit', + url: 'https://some-post-url', + post: { + title: 'post title', + body: 'post body', + }, + postedAt: '2020-05-27T15:13:30Z', + tenantId: mockIRepositoryOptions.currentTenant.id, + } as EagleEyeContent + + const contentCreated = await EagleEyeContentRepository.create(content, mockIRepositoryOptions) + + const action: EagleEyeAction = { + type: EagleEyeActionType.BOOKMARK, + timestamp: '2022-07-27T19:13:30Z', + } + + const actionCreated = await EagleEyeActionRepository.createActionForContent( + action, + contentCreated.id, + mockIRepositoryOptions, + ) + + actionCreated.createdAt = (actionCreated.createdAt as Date).toISOString().split('T')[0] + actionCreated.updatedAt = (actionCreated.updatedAt as Date).toISOString().split('T')[0] + + const expectedAction = { + id: actionCreated.id, + ...action, + timestamp: new Date(actionCreated.timestamp), + contentId: contentCreated.id, + actionById: mockIRepositoryOptions.currentUser.id, + tenantId: mockIRepositoryOptions.currentTenant.id, + createdAt: SequelizeTestUtils.getNowWithoutTime(), + updatedAt: SequelizeTestUtils.getNowWithoutTime(), + } + expect(expectedAction).toStrictEqual(actionCreated) + }) + }) +}) diff --git a/backend/src/database/repositories/__tests__/eagleEyeContentRepository.test.ts b/backend/src/database/repositories/__tests__/eagleEyeContentRepository.test.ts index bffec870ca..96666a7148 100644 --- a/backend/src/database/repositories/__tests__/eagleEyeContentRepository.test.ts +++ b/backend/src/database/repositories/__tests__/eagleEyeContentRepository.test.ts @@ -1,86 +1,10 @@ -import lodash from 'lodash' -import moment from 'moment' import EagleEyeContentRepository from '../eagleEyeContentRepository' import SequelizeTestUtils from '../../utils/sequelizeTestUtils' -import Error404 from '../../../errors/Error404' -import Error400 from '../../../errors/Error400' -import EagleEyeContentService from '../../../services/eagleEyeContentService' -import { PlatformType } from '../../../types/integrationEnums' +import { EagleEyeActionType, EagleEyeContent } from '../../../types/eagleEyeTypes' +import EagleEyeActionRepository from '../eagleEyeActionRepository' const db = null -const toCreate = { - sourceId: 'sourceId', - vectorId: '123', - status: null, - platform: 'hacker_news', - title: 'title', - userAttributes: { [PlatformType.GITHUB]: 'hey', [PlatformType.TWITTER]: 'ho' }, - text: 'text', - postAttributes: { - score: 10, - }, - url: 'url', - exactKeywords: null, - timestamp: new Date(), - username: 'username', - keywords: ['keyword1', 'keyword2'], - similarityScore: 0.9, -} - -const toCreateMinimal = { - sourceId: 'sourceIdMinimal', - vectorId: '456', - platform: 'hacker_news', - url: 'url', - title: 'title minimal', - timestamp: new Date(), - username: 'username', - keywords: 'keyword', -} - -const forFiltering = [ - toCreate, - toCreateMinimal, - { - sourceId: 'devto123', - vectorId: '123123', - status: 'engaged', - url: 'devto url', - username: 'devtousername1', - platform: 'devto', - timestamp: moment().toDate(), - title: 'title devto 1', - }, - { - sourceId: 'devto456', - vectorId: '123456', - url: 'url devto 2', - username: 'devtousername2', - status: 'rejected', - platform: 'devto', - timestamp: moment().subtract(1, 'week').toDate(), - title: 'title devto 2', - keywords: ['keyword1', 'keyword2'], - score: 40, - }, - { - sourceId: 'devto789', - vectorId: '123456', - url: 'url devto 3', - username: 'devtousername3', - status: null, - platform: 'devto', - timestamp: moment().subtract(1, 'week').toDate(), - keywords: ['keyword3', 'keyword2'], - title: 'title devto 3', - }, -] - -async function addAll(options) { - await Promise.all(forFiltering.map((item) => EagleEyeContentRepository.upsert(item, options))) -} - describe('eagleEyeContentRepository tests', () => { beforeEach(async () => { await SequelizeTestUtils.wipeDatabase(db) @@ -92,29 +16,40 @@ describe('eagleEyeContentRepository tests', () => { done() }) - describe('upserts method', () => { - it('Should create a complete content succesfully', async () => { + describe('create method', () => { + it('Should create a content succesfully', async () => { const mockIRepositoryOptions = await SequelizeTestUtils.getTestIRepositoryOptions(db) - const created = await EagleEyeContentRepository.upsert(toCreate, mockIRepositoryOptions) + const content = { + platform: 'reddit', + url: 'https://some-post-url', + post: { + title: 'post title', + body: 'post body', + }, + postedAt: '2020-05-27T15:13:30Z', + tenantId: mockIRepositoryOptions.currentTenant.id, + } as EagleEyeContent - created.createdAt = created.createdAt.toISOString().split('T')[0] - created.updatedAt = created.updatedAt.toISOString().split('T')[0] + const created = await EagleEyeContentRepository.create(content, mockIRepositoryOptions) + + created.createdAt = (created.createdAt as Date).toISOString().split('T')[0] + created.updatedAt = (created.updatedAt as Date).toISOString().split('T')[0] const expectedCreated = { id: created.id, - ...toCreate, - importHash: null, + ...content, + postedAt: new Date(content.postedAt), + actions: [], createdAt: SequelizeTestUtils.getNowWithoutTime(), updatedAt: SequelizeTestUtils.getNowWithoutTime(), - deletedAt: null, tenantId: mockIRepositoryOptions.currentTenant.id, - createdById: mockIRepositoryOptions.currentUser.id, - updatedById: mockIRepositoryOptions.currentUser.id, } expect(created).toStrictEqual(expectedCreated) }) + /* + it('Should create a content with unix timestamp', async () => { const mockIRepositoryOptions = await SequelizeTestUtils.getTestIRepositoryOptions(db) @@ -157,130 +92,9 @@ describe('eagleEyeContentRepository tests', () => { expect(created).toStrictEqual(expectedCreated) }) - it('Should not add it the record already exists', async () => { - const mockIRepositoryOptions = await SequelizeTestUtils.getTestIRepositoryOptions(db) - - await EagleEyeContentRepository.upsert(toCreate, mockIRepositoryOptions) - - await EagleEyeContentRepository.upsert(toCreate, mockIRepositoryOptions) - - const count = await mockIRepositoryOptions.database.eagleEyeContent.count() - expect(count).toBe(1) - }) - - it('Should update keywords and similarity score if the item already exists', async () => { - const mockIRepositoryOptions = await SequelizeTestUtils.getTestIRepositoryOptions(db) - - await EagleEyeContentRepository.upsert(toCreate, mockIRepositoryOptions) - - const toCreateNewKeywords = { ...toCreate } - toCreateNewKeywords.keywords = ['1', '2', 'keyword1'] - toCreateNewKeywords.similarityScore = 0.95 - - const allKeywords = ['1', '2', 'keyword1', 'keyword2'] - - const created = await EagleEyeContentRepository.upsert( - toCreateNewKeywords, - mockIRepositoryOptions, - ) - - const count = await mockIRepositoryOptions.database.eagleEyeContent.count() - expect(count).toBe(1) - expect(lodash.isEqual(created.keywords.sort(), allKeywords.sort())) - expect(created.similarityScore).toBe(0.95) - }) - - it('Should create a minimal content succesfully', async () => { - const mockIRepositoryOptions = await SequelizeTestUtils.getTestIRepositoryOptions(db) - - const created = await EagleEyeContentRepository.upsert( - toCreateMinimal, - mockIRepositoryOptions, - ) - - created.createdAt = created.createdAt.toISOString().split('T')[0] - created.updatedAt = created.updatedAt.toISOString().split('T')[0] - - const expectedCreated = { - id: created.id, - ...toCreateMinimal, - text: null, - status: null, - userAttributes: null, - postAttributes: null, - similarityScore: null, - exactKeywords: null, - importHash: null, - createdAt: SequelizeTestUtils.getNowWithoutTime(), - updatedAt: SequelizeTestUtils.getNowWithoutTime(), - deletedAt: null, - tenantId: mockIRepositoryOptions.currentTenant.id, - createdById: mockIRepositoryOptions.currentUser.id, - updatedById: mockIRepositoryOptions.currentUser.id, - } - expect(created).toStrictEqual(expectedCreated) - expect(created.status).toBe(null) - }) - - it('Should create with rejected status', async () => { - const mockIRepositoryOptions = await SequelizeTestUtils.getTestIRepositoryOptions(db) - const newStatus = { ...toCreate } - newStatus.status = 'rejected' - - const created = await EagleEyeContentRepository.upsert(newStatus, mockIRepositoryOptions) - - created.createdAt = created.createdAt.toISOString().split('T')[0] - created.updatedAt = created.updatedAt.toISOString().split('T')[0] + - const expectedCreated = { - id: created.id, - ...newStatus, - importHash: null, - createdAt: SequelizeTestUtils.getNowWithoutTime(), - updatedAt: SequelizeTestUtils.getNowWithoutTime(), - deletedAt: null, - tenantId: mockIRepositoryOptions.currentTenant.id, - createdById: mockIRepositoryOptions.currentUser.id, - updatedById: mockIRepositoryOptions.currentUser.id, - } - expect(created).toStrictEqual(expectedCreated) - expect(created.status).toBe('rejected') - }) - - it('Should create with engaged status', async () => { - const mockIRepositoryOptions = await SequelizeTestUtils.getTestIRepositoryOptions(db) - const newStatus = { ...toCreate } - newStatus.status = 'engaged' - - const created = await EagleEyeContentRepository.upsert(newStatus, mockIRepositoryOptions) - - created.createdAt = created.createdAt.toISOString().split('T')[0] - created.updatedAt = created.updatedAt.toISOString().split('T')[0] - - const expectedCreated = { - id: created.id, - ...newStatus, - importHash: null, - createdAt: SequelizeTestUtils.getNowWithoutTime(), - updatedAt: SequelizeTestUtils.getNowWithoutTime(), - deletedAt: null, - tenantId: mockIRepositoryOptions.currentTenant.id, - createdById: mockIRepositoryOptions.currentUser.id, - updatedById: mockIRepositoryOptions.currentUser.id, - } - expect(created).toStrictEqual(expectedCreated) - expect(created.status).toBe('engaged') - }) - - it('Should throw an error for an invalid status', async () => { - const mockIRepositoryOptions = await SequelizeTestUtils.getTestIRepositoryOptions(db) - const newStatus = { ...toCreate } - newStatus.status = 'smth else' - - await expect(() => - EagleEyeContentRepository.upsert(newStatus, mockIRepositoryOptions), - ).rejects.toThrowError(new Error400('en', 'errors.invalidEagleEyeStatus.message')) - }) + }) describe('find by id method', () => { @@ -601,4 +415,138 @@ describe('eagleEyeContentRepository tests', () => { expect(updated.keywords).toStrictEqual(created.keywords) }) }) + */ + }) + + describe('findAndCountAll method', () => { + it('Should find eagle eye contant, various cases', async () => { + // create random tenant with one user + const mockIRepositoryOptions = await SequelizeTestUtils.getTestIRepositoryOptions(db) + + // create additional users for same tenant to test out actionBy filtering + const randomUser = await SequelizeTestUtils.getRandomUser() + + console.log('random user: ') + console.log(randomUser) + + const user2 = await mockIRepositoryOptions.database.user.create(randomUser) + + await mockIRepositoryOptions.database.tenantUser.create({ + roles: ['admin'], + status: 'active', + tenantId: mockIRepositoryOptions.currentTenant.id, + userId: user2.id, + }) + + // create few content + // one without any actions + await EagleEyeContentRepository.create( + { + platform: 'reddit', + url: 'https://some-reddit-url', + post: { + title: 'post title', + body: 'post body', + }, + postedAt: '2020-05-27T15:13:30Z', + tenantId: mockIRepositoryOptions.currentTenant.id, + }, + mockIRepositoryOptions, + ) + + // one with a bookmark action + let c2 = await EagleEyeContentRepository.create( + { + platform: 'hackernews', + url: 'https://some-hackernews-url', + post: { + title: 'post title', + body: 'post body', + }, + postedAt: '2022-06-27T19:14:44Z', + tenantId: mockIRepositoryOptions.currentTenant.id, + }, + mockIRepositoryOptions, + ) + + // add bookmark action + await EagleEyeActionRepository.createActionForContent( + { + type: EagleEyeActionType.BOOKMARK, + timestamp: '2022-07-27T19:13:30Z', + }, + c2.id, + mockIRepositoryOptions, + ) + + c2 = await EagleEyeContentRepository.findById(c2.id, mockIRepositoryOptions) + + // another content with a thumbs-up(user1) and a bookmark(user2) action + let c3 = await EagleEyeContentRepository.create( + { + platform: 'devto', + url: 'https://some-devto-url', + post: { + title: 'post title', + body: 'post body', + }, + postedAt: '2022-06-27T19:14:44Z', + tenantId: mockIRepositoryOptions.currentTenant.id, + }, + mockIRepositoryOptions, + ) + + // add the thumbs up action + await EagleEyeActionRepository.createActionForContent( + { + type: EagleEyeActionType.THUMBS_UP, + timestamp: '2022-09-30T23:11:10Z', + }, + c3.id, + mockIRepositoryOptions, + ) + + // also add bookmark from user2 + await EagleEyeActionRepository.createActionForContent( + { + type: EagleEyeActionType.BOOKMARK, + timestamp: '2022-09-30T23:11:10Z', + }, + c3.id, + { ...mockIRepositoryOptions, currentUser: user2 }, + ) + + c3 = await EagleEyeContentRepository.findById(c3.id, mockIRepositoryOptions) + + // filter by action type + let res = await EagleEyeContentRepository.findAndCountAll( + { + advancedFilter: { + action: { + type: EagleEyeActionType.BOOKMARK, + }, + }, + }, + mockIRepositoryOptions, + ) + + expect(res.count).toBe(2) + expect(res.rows.sort((a, b) => (a.createdAt > b.createdAt ? 1 : -1))).toStrictEqual([c2, c3]) + + // filter by actionBy + res = await EagleEyeContentRepository.findAndCountAll( + { + advancedFilter: { + action: { + actionById: user2.id, + }, + }, + }, + mockIRepositoryOptions, + ) + + expect(res.count).toBe(1) + expect(res.rows).toStrictEqual([c3]) + }) + }) }) diff --git a/backend/src/database/repositories/eagleEyeActionRepository.ts b/backend/src/database/repositories/eagleEyeActionRepository.ts new file mode 100644 index 0000000000..cc7932b257 --- /dev/null +++ b/backend/src/database/repositories/eagleEyeActionRepository.ts @@ -0,0 +1,125 @@ +import lodash from 'lodash' +import Error404 from '../../errors/Error404' +import { EagleEyeAction, EagleEyeActionType } from '../../types/eagleEyeTypes' +import { IRepositoryOptions } from './IRepositoryOptions' +import SequelizeRepository from './sequelizeRepository' + +export default class EagleEyeActionRepository { + static async createActionForContent( + data: EagleEyeAction, + contentId: string, + options: IRepositoryOptions, + ): Promise { + const currentUser = SequelizeRepository.getCurrentUser(options) + + const transaction = SequelizeRepository.getTransaction(options) + + const currentTenant = SequelizeRepository.getCurrentTenant(options) + + const record = await options.database.eagleEyeAction.create( + { + ...lodash.pick(data, ['type', 'timestamp']), + actionById: currentUser.id, + contentId, + tenantId: currentTenant.id, + }, + { + transaction, + }, + ) + + return this.findById(record.id, options) + } + + static async removeActionFromContent( + action: EagleEyeActionType, + contentId: string, + options: IRepositoryOptions, + ) { + const currentUser = SequelizeRepository.getCurrentUser(options) + + const currentTenant = SequelizeRepository.getCurrentTenant(options) + + const transaction = SequelizeRepository.getTransaction(options) + + const record = await options.database.eagleEyeAction.findOne({ + where: { + contentId, + action, + actionById: currentUser.id, + tenantId: currentTenant.id, + }, + transaction, + }) + + if (record) { + await record.destroy({ + transaction, + force: true, + }) + } + } + + static async destroy(id: string, options: IRepositoryOptions): Promise { + const transaction = SequelizeRepository.getTransaction(options) + + const currentTenant = SequelizeRepository.getCurrentTenant(options) + + const record = await options.database.eagleEyeAction.findOne({ + where: { + id, + tenantId: currentTenant.id, + }, + transaction, + }) + + if (record) { + await record.destroy({ + transaction, + force: true, + }) + } + } + + static async findById(id: string, options: IRepositoryOptions) { + const transaction = SequelizeRepository.getTransaction(options) + + const include = [] + + const currentTenant = SequelizeRepository.getCurrentTenant(options) + + const record = await options.database.eagleEyeAction.findOne({ + where: { + id, + tenantId: currentTenant.id, + }, + include, + transaction, + }) + + if (!record) { + throw new Error404() + } + + return this._populateRelations(record) + } + + static async create(data: EagleEyeAction, options: IRepositoryOptions): Promise { + const currentTenant = SequelizeRepository.getCurrentTenant(options) + + const record = options.database.eagleEyeContent.create({ + ...lodash.pick(data, ['type', 'timestamp']), + tenantId: currentTenant.id, + }) + + return this.findById(record.id, options) + } + + static async _populateRelations(record) { + if (!record) { + return record + } + + return record.get({ plain: true }) + } +} diff --git a/backend/src/database/repositories/eagleEyeContentRepository.ts b/backend/src/database/repositories/eagleEyeContentRepository.ts index 56de3e60f0..9a34d4d9d4 100644 --- a/backend/src/database/repositories/eagleEyeContentRepository.ts +++ b/backend/src/database/repositories/eagleEyeContentRepository.ts @@ -1,101 +1,56 @@ -import moment from 'moment' import lodash from 'lodash' +import { Op } from 'sequelize' import SequelizeRepository from './sequelizeRepository' import Error404 from '../../errors/Error404' -import Error400 from '../../errors/Error400' -import AuditLogRepository from './auditLogRepository' import { IRepositoryOptions } from './IRepositoryOptions' +import { EagleEyeContent } from '../../types/eagleEyeTypes' import QueryParser from './filters/queryParser' import { QueryOutput } from './filters/queryTypes' +import EagleEyeActionRepository from './eagleEyeActionRepository' export default class EagleEyeContentRepository { - /** - * Create an eagle eye shown content record. - * @param data Data to a new EagleEyeContent record. - * @param options Repository options. - * @returns Created EagleEyeContent record. - */ - static async upsert(data, options: IRepositoryOptions) { - const currentUser = SequelizeRepository.getCurrentUser(options) + static async create( + data: EagleEyeContent, + options: IRepositoryOptions, + ): Promise { + const currentTenant = SequelizeRepository.getCurrentTenant(options) + + const record = await options.database.eagleEyeContent.create({ + ...lodash.pick(data, ['platform', 'post', 'url', 'postedAt']), + tenantId: currentTenant.id, + }) + + if (data.actions) { + for (const action of data.actions) { + await EagleEyeActionRepository.createActionForContent(action, record.id, options) + } + } + + return this.findById(record.id, options) + } - const tenant = SequelizeRepository.getCurrentTenant(options) + static async update(id, data, options: IRepositoryOptions) { + const currentUser = SequelizeRepository.getCurrentUser(options) const transaction = SequelizeRepository.getTransaction(options) - if (data.status && ![null, 'rejected', 'engaged'].includes(data.status)) { - throw new Error400('en', 'errors.invalidEagleEyeStatus.message') - } + const currentTenant = SequelizeRepository.getCurrentTenant(options) - const existing = await options.database.eagleEyeContent.findOne({ + let record = await options.database.eagleEyeContent.findOne({ where: { - tenantId: tenant.id, - sourceId: data.sourceId, + id, + tenantId: currentTenant.id, }, + transaction, }) - // If the content is already shown, we don't need to add it again - if (existing) { - // If the content comes from a different kewword, we also add it - if (!lodash.isEqual(data.keywords.sort(), existing.keywords.sort())) { - const keywords = lodash.uniq([...existing.keywords, ...data.keywords]) - let exactKeywords = null - if (data.exactKeywords && !existing.exactKeywords) { - exactKeywords = data.exactKeywords - } else if (!data.exactKeywords && existing.exactKeywords) { - exactKeywords = existing.exactKeywords - } - if (data.exactKeywords && existing.exactKeywords) { - exactKeywords = lodash.uniq([...existing.exactKeywords, ...data.exactKeywords]) - } - const similarityScore = data.similarityScore - return existing.update( - { - keywords, - similarityScore, - exactKeywords, - }, - { - transaction, - }, - ) - } - return existing - } - - if (typeof data.keywords === 'string') { - data.keywords = [data.keywords] - } - - if (typeof data.exactKeywords === 'string') { - data.exactKeywords = [data.exactKeywords] - } - - if (typeof data.timestamp === 'number') { - data.timestamp = moment.unix(data.timestamp).toDate() + if (!record) { + throw new Error404() } - const record = await options.database.eagleEyeContent.create( + record = await record.update( { - ...lodash.pick(data, [ - 'sourceId', - 'vectorId', - 'status', - 'postAttributes', - 'title', - 'username', - 'url', - 'text', - 'timestamp', - 'userAttributes', - 'platform', - 'keywords', - 'exactKeywords', - 'similarityScore', - 'importHash', - ]), - - tenantId: tenant.id, - createdById: currentUser.id, + ...lodash.pick(data, ['platform', 'post', 'postedAt', 'url']), updatedById: currentUser.id, }, { @@ -103,192 +58,87 @@ export default class EagleEyeContentRepository { }, ) - await AuditLogRepository.log( - { - entityName: 'eagleEyeContent', - entityId: record.id, - action: AuditLogRepository.CREATE, - values: data, - }, - options, - ) - return this.findById(record.id, options) } - /** - * EagleEyeContent find all records matching given criteria. - * @returns Records found. - */ - static async findAndCountAll( - { filter = {} as any, advancedFilter = null as any, limit = 0, offset = 0, orderBy = '' }, - options: IRepositoryOptions, - ) { - // If the advanced filter is empty, we construct it from the query parameter filter - if (!advancedFilter) { - advancedFilter = { and: [] } - - if (filter.id) { - advancedFilter.and.push({ - id: filter.id, - }) - } + static async findById(id: string, options: IRepositoryOptions) { + const transaction = SequelizeRepository.getTransaction(options) - if (filter.sourceId) { - advancedFilter.and.push({ - sourceId: filter.sourceId, - }) - } + const include = [ + { + model: options.database.eagleEyeAction, + as: 'actions', + }, + ] - if (filter.vectorId) { - advancedFilter.and.push({ - vectorId: filter.vectorId, - }) - } + const currentTenant = SequelizeRepository.getCurrentTenant(options) - if (filter.status) { - if (filter.status === 'NULL') { - advancedFilter.and.push({ - status: 'NULL', - }) - } else if (filter.status === 'NOT_NULL') { - advancedFilter.and.push({ - status: { - not: null, - }, - }) - } else { - advancedFilter.and.push({ - status: { - textContains: filter.status, - }, - }) - } - } + const record = await options.database.eagleEyeContent.findOne({ + where: { + id, + tenantId: currentTenant.id, + }, + include, + transaction, + }) - if (filter.timestampRange) { - const [start, end] = filter.timestampRange - - if (start !== undefined && start !== null && start !== '') { - advancedFilter.and.push({ - timestamp: { - gte: start, - }, - }) - } - - if (end !== undefined && end !== null && end !== '') { - advancedFilter.and.push({ - timestamp: { - lte: end, - }, - }) - } - } + if (!record) { + throw new Error404() + } - if (filter.platforms) { - advancedFilter.and.push({ - platform: { - or: filter.platforms.split(','), - }, - }) - } + return this._populateRelations(record) + } - if (filter.nDays) { - advancedFilter.and.push({ - timestamp: { - gte: moment().subtract(filter.nDays, 'days').toDate(), - }, - }) - } + static async destroy(id: string, options: IRepositoryOptions): Promise { + const transaction = SequelizeRepository.getTransaction(options) - if (filter.title) { - advancedFilter.and.push({ - title: { - textContains: filter.title, - }, - }) - } + const currentTenant = SequelizeRepository.getCurrentTenant(options) - if (filter.text) { - advancedFilter.and.push({ - text: { - textContains: filter.text, - }, - }) - } + const record = await options.database.eagleEyeContent.findOne({ + where: { + id, + tenantId: currentTenant.id, + }, + transaction, + }) - if (filter.url) { - advancedFilter.and.push({ - url: { - textContains: filter.url, - }, - }) - } + if (record) { + await record.destroy({ + transaction, + force: true, + }) + } + } - if (filter.username) { - advancedFilter.and.push({ - username: { - textContains: filter.username, - }, - }) - } + static async findAndCountAll( + { advancedFilter = null as any, limit = 0, offset = 0, orderBy = '' }, + options: IRepositoryOptions, + ) { + const actionsSequelizeInclude = { + model: options.database.eagleEyeAction, + as: 'actions', + where: {}, + } - if (filter.keywords) { - // Overlap will take a post where any keyword matches any of the filter keywords - advancedFilter.and.push({ - keywords: { - overlap: filter.keywords.split(','), - }, - }) - } + if (advancedFilter && advancedFilter.action) { + const actionQueryParser = new QueryParser({}, options) - if (filter.similarityScoreRange) { - const [start, end] = filter.similarityScoreRange - - if (start !== undefined && start !== null && start !== '') { - advancedFilter.and.push({ - similarityScore: { - gte: start, - }, - }) - } - - if (end !== undefined && end !== null && end !== '') { - advancedFilter.and.push({ - similarityScore: { - lte: end, - }, - }) - } - } + const parsedActionQuery: QueryOutput = actionQueryParser.parse({ + filter: advancedFilter.action, + orderBy: 'timestamp_DESC', + }) - if (filter.createdAtRange) { - const [start, end] = filter.createdAtRange - - if (start !== undefined && start !== null && start !== '') { - advancedFilter.and.push({ - createdAt: { - gte: start, - }, - }) - } - - if (end !== undefined && end !== null && end !== '') { - advancedFilter.and.push({ - createdAt: { - lte: end, - }, - }) - } - } + actionsSequelizeInclude.where = parsedActionQuery.where ?? {} + delete advancedFilter.action } - const parser = new QueryParser({}, options) + const include = [actionsSequelizeInclude] + + const contentParser = new QueryParser({}, options) - const parsed: QueryOutput = parser.parse({ + const parsed: QueryOutput = contentParser.parse({ filter: advancedFilter, - orderBy: orderBy || ['createdAt_DESC'], + orderBy: orderBy || ['postedAt_DESC'], limit, offset, }) @@ -297,79 +147,38 @@ export default class EagleEyeContentRepository { rows, count, // eslint-disable-line prefer-const } = await options.database.eagleEyeContent.findAndCountAll({ + include, ...(parsed.where ? { where: parsed.where } : {}), - ...(parsed.having ? { having: parsed.having } : {}), order: parsed.order, limit: parsed.limit, offset: parsed.offset, transaction: SequelizeRepository.getTransaction(options), + subQuery: false, }) - rows = await this._populateRelationsForRows(rows) - - return { rows, count, limit: parsed.limit, offset: parsed.offset } - } - - static async update(id, data, options: IRepositoryOptions) { - const currentUser = SequelizeRepository.getCurrentUser(options) - - const transaction = SequelizeRepository.getTransaction(options) - - const currentTenant = SequelizeRepository.getCurrentTenant(options) - - let record = await options.database.eagleEyeContent.findOne({ - where: { - id, - tenantId: currentTenant.id, - }, - transaction, - }) - - if (!record) { - throw new Error404() - } - - if (data.status && ![null, 'rejected', 'engaged'].includes(data.status)) { - throw new Error400('en', 'errors.invalidEagleEyeStatus.message') + // If we have an actions filter, we should query again to eager + // load the all actions on a content because previous query will + // omit actions that don't match the given action filter + if (Object.keys(actionsSequelizeInclude.where).length !== 0) { + rows = ( + await options.database.eagleEyeContent.findAndCountAll({ + include: [{ ...actionsSequelizeInclude, where: {} }], + where: { id: { [Op.in]: rows.map((i) => i.id) } }, + order: parsed.order, + limit: parsed.limit, + offset: parsed.offset, + transaction: SequelizeRepository.getTransaction(options), + subQuery: false, + }) + ).rows } - record = await record.update( - { - ...lodash.pick(data, [ - 'sourceId', - 'vectorId', - 'status', - 'title', - 'username', - 'url', - 'text', - 'postAttributes', - 'timestamp', - 'platform', - 'userAttributes', - 'importHash', - // Missing keywords on purpose - ]), - updatedById: currentUser.id, - }, - { - transaction, - }, - ) + rows = await this._populateRelationsForRows(rows) - await AuditLogRepository.log( - { - entityName: 'eagleEyeContent', - entityId: record.id, - action: AuditLogRepository.UPDATE, - values: data, - }, - options, - ) - return this.findById(record.id, options) + return { rows, count, limit: parsed.limit, offset: parsed.offset } } - static async findById(id, options: IRepositoryOptions) { + static async findByUrl(url: string, options: IRepositoryOptions) { const transaction = SequelizeRepository.getTransaction(options) const include = [] @@ -378,7 +187,7 @@ export default class EagleEyeContentRepository { const record = await options.database.eagleEyeContent.findOne({ where: { - id, + url, tenantId: currentTenant.id, }, include, @@ -386,7 +195,7 @@ export default class EagleEyeContentRepository { }) if (!record) { - throw new Error404() + return null } return this._populateRelations(record) diff --git a/backend/src/database/repositories/userRepository.ts b/backend/src/database/repositories/userRepository.ts index 8b80aee5ed..781e93d1cf 100644 --- a/backend/src/database/repositories/userRepository.ts +++ b/backend/src/database/repositories/userRepository.ts @@ -320,6 +320,37 @@ export default class UserRepository { return this.findById(user.id, options) } + static async updateEagleEyeSettings(id: string, data, options: IRepositoryOptions) { + const currentUser = SequelizeRepository.getCurrentUser(options) + const transaction = SequelizeRepository.getTransaction(options) + + const user = await options.database.user.findByPk(id, { + transaction, + }) + + await user.update( + { + eagleEyeSettings: data, + updatedById: currentUser.id, + }, + { transaction }, + ) + + await AuditLogRepository.log( + { + entityName: 'user', + entityId: user.id, + action: AuditLogRepository.UPDATE, + values: { + ...user.get({ plain: true }), + eagleEyeSettings: data, + }, + }, + options, + ) + return this.findById(user.id, options) + } + static async findByEmail(email, options: IRepositoryOptions) { const transaction = SequelizeRepository.getTransaction(options) diff --git a/backend/src/database/utils/sequelizeTestUtils.ts b/backend/src/database/utils/sequelizeTestUtils.ts index 7b103981f4..83e813d9a2 100644 --- a/backend/src/database/utils/sequelizeTestUtils.ts +++ b/backend/src/database/utils/sequelizeTestUtils.ts @@ -34,6 +34,7 @@ export default class SequelizeTestUtils { files, microservices, "eagleEyeContents", + "eagleEyeActions", "auditLogs", "memberEnrichmentCache" cascade; diff --git a/backend/src/i18n/en.ts b/backend/src/i18n/en.ts index 0a41b33134..7df609a98f 100644 --- a/backend/src/i18n/en.ts +++ b/backend/src/i18n/en.ts @@ -102,19 +102,29 @@ const en = { activityDup: { message: 'This activity has already been linked to this member', }, - invalidEagleEyeStatus: { - message: 'Possible statuses are: "shown", "rejected", "engaged"', - }, - eagleEyeSearchFailed: { - message: 'Search failed in EagleEye', - }, OrganizationNameRequired: { message: 'Organization Name is required', }, projectNotFound: { message: 'Project not found', }, - + eagleEye: { + urlRequiredWhenUpserting: 'URL field is mandatory when upserting eagleEyeContent', + contentNotFound: 'Eagle eye content not found. Action will not be created.', + feedSettingsMissing: 'Feed settings are missing. Settings not updated.', + keywordsMissing: + 'Either keywords or exactKeywords are required in feeds. Settings not updated.', + platformMissing: + 'feed.platforms is required and must be a non-empty list. Settings not updated.', + platformInvalid: `feed.platforms contains {0}, which is not in [{1}]. Settings not updated.`, + publishedDateMissing: + 'feed.publishedDate is missing or invalid. It should be one of [{0}]. Settings not updated.', + emailInvalid: 'emailDigest.email needs a valid email address. Settings not updated.', + frequencyInvalid: + 'emailDigest.frequency needs to be one of daily, weekly. Settings not updated.', + timeInvalid: 'emailDigest.time needs to be a valid time. Settings not updated.', + notOnboarded: 'Eagle eye is not set up yet.', + }, integrations: { badEndpoint: 'Bad endpoint: {0}', }, @@ -195,6 +205,12 @@ const en = { planLimitExceeded: 'You have exceeded # of automations you can have in your plan.', }, }, + eagleEye: { + errors: { + planLimitExceeded: + 'EagleEye is only available in the Growth and Custom plans. Please upgrade your plan.', + }, + }, }, communityHelpCenter: { diff --git a/backend/src/security/permissions.ts b/backend/src/security/permissions.ts index 3c75df4c2b..ec28c71024 100644 --- a/backend/src/security/permissions.ts +++ b/backend/src/security/permissions.ts @@ -422,6 +422,21 @@ class Permissions { allowedRoles: [roles.admin, roles.readonly], allowedPlans: [plans.essential, plans.growth], }, + eagleEyeActionCreate: { + id: 'eagleEyeActionCreate', + allowedRoles: [roles.admin], + allowedPlans: [plans.growth], + }, + eagleEyeActionDestroy: { + id: 'eagleEyeActionDestroy', + allowedRoles: [roles.admin], + allowedPlans: [plans.growth], + }, + eagleEyeContentCreate: { + id: 'eagleEyeContentCreate', + allowedRoles: [roles.admin], + allowedPlans: [plans.growth], + }, eagleEyeContentRead: { id: 'eagleEyeContentRead', allowedRoles: [roles.admin, roles.readonly], diff --git a/backend/src/serverless/integrations/services/integrations/hackerNewsIntegrationService.ts b/backend/src/serverless/integrations/services/integrations/hackerNewsIntegrationService.ts index f454b6171b..254125c88e 100644 --- a/backend/src/serverless/integrations/services/integrations/hackerNewsIntegrationService.ts +++ b/backend/src/serverless/integrations/services/integrations/hackerNewsIntegrationService.ts @@ -1,4 +1,5 @@ import sanitizeHtml from 'sanitize-html' +import moment from 'moment' import { MemberAttributeName } from '../../../../database/attributes/member/enums' import { HackerNewsMemberAttributes } from '../../../../database/attributes/member/hackerNews' import MemberAttributeSettingsService from '../../../../services/memberAttributeSettingsService' @@ -13,9 +14,9 @@ import Operations from '../../../dbOperations/operations' import getPost from '../../usecases/hackerNews/getPost' import { HackerNewsGrid } from '../../grid/hackerNewsGrid' import { - EagleEyeResponse, HackerNewsResponse, HackerNewsIntegrationSettings, + HackerNewsSearchResult, } from '../../types/hackerNewsTypes' import { AddActivitiesSingle } from '../../types/messageTypes' import getPostsByKeywords from '../../usecases/hackerNews/getPostsByKeywords' @@ -43,7 +44,10 @@ export class HackerNewsIntegrationService extends IntegrationServiceBase { const keywords = Array.from(new Set([...settings.keywords, ...settings.urls])) this.logger(context).info(`Fetching posts for keywords: ${keywords}`) const posts = await getPostsByKeywords( - { keywords, nDays: context.onboarding ? 1000000 : 3 }, + { + keywords, + after: context.onboarding ? 0 : moment().subtract(30, 'days').unix(), + }, context.serviceContext, this.logger(context), ) @@ -55,8 +59,8 @@ export class HackerNewsIntegrationService extends IntegrationServiceBase { } async getStreams(context: IStepContext): Promise { - return context.pipelineData.posts.map((a: EagleEyeResponse) => ({ - value: a.sourceId.slice(a.sourceId.lastIndexOf(':') + 1), + return context.pipelineData.posts.map((a: HackerNewsSearchResult) => ({ + value: a.postId, metadata: { channel: a.keywords[0], }, diff --git a/backend/src/serverless/integrations/types/hackerNewsTypes.ts b/backend/src/serverless/integrations/types/hackerNewsTypes.ts index 89bc808f3a..5093679316 100644 --- a/backend/src/serverless/integrations/types/hackerNewsTypes.ts +++ b/backend/src/serverless/integrations/types/hackerNewsTypes.ts @@ -1,22 +1,37 @@ -export interface EagleEyeResponse { - vectorId: number - sourceId: string - title: string - url: string - createdAt: string - text: string - username: string - platform: string - timestamp: string - userAttributes: any - postAttributes: { - commentsCount: number - score: number - } - keywords: string[] +export interface HackerNewsSearchResponseRaw { + hits: [ + { + created_at: string + title: string + url: string + author: string + points: number + story_text: string + comment_text: string | null + num_comments: number + story_id: string | null + story_title: string | null + story_url: string | null + parent_id: string | null + created_at_i: number + _tags: string[] + objectID: string + _highlightResult: { + [key: string]: { + value: string + matchLevel: string + matchedWords: string[] + fullyHighlighted?: boolean + } + } + }, + ] } -export type EagleEyeResponses = EagleEyeResponse[] +export interface HackerNewsSearchResult { + keywords: string[] + postId: number +} export interface HackerNewsPost { by: string @@ -49,7 +64,7 @@ export interface HackerNewsIntegrationSettings { urls: string[] } -export interface EagleEyeInput { +export interface HackerNewsKeywordSearchInput { keywords: string[] - nDays: number + after: number } diff --git a/backend/src/serverless/integrations/usecases/hackerNews/getPostsByKeywords.ts b/backend/src/serverless/integrations/usecases/hackerNews/getPostsByKeywords.ts index 0e19a3ac5e..6a3563b131 100644 --- a/backend/src/serverless/integrations/usecases/hackerNews/getPostsByKeywords.ts +++ b/backend/src/serverless/integrations/usecases/hackerNews/getPostsByKeywords.ts @@ -1,23 +1,50 @@ +import axios from 'axios' import { IServiceOptions } from '../../../../services/IServiceOptions' -import { EagleEyeResponses, EagleEyeInput } from '../../types/hackerNewsTypes' +import { + HackerNewsKeywordSearchInput, + HackerNewsSearchResponseRaw, + HackerNewsSearchResult, +} from '../../types/hackerNewsTypes' import { Logger } from '../../../../utils/logging' import { timeout } from '../../../../utils/timing' -import EagleEyeContentService from '../../../../services/eagleEyeContentService' async function getPostsByKeyword( - input: EagleEyeInput, + input: HackerNewsKeywordSearchInput, options: IServiceOptions, logger: Logger, -): Promise { +): Promise { await timeout(2000) try { - const eagleEyeService = new EagleEyeContentService(options) - return await eagleEyeService.keywordMatch({ - keywords: input.keywords, - nDays: input.nDays, - platform: 'hacker_news', - }) + const out = [] + const existing = new Set() + for (const keyword of input.keywords) { + const config = { + method: 'get', + maxBodyLength: Infinity, + url: 'http://hn.algolia.com/api/v1/search', + params: { + query: keyword, + numericFilters: `created_at_i>${input.after}`, + tags: '(story,ask_hn,show_hn,poll)', + }, + headers: {}, + } + + const response = await axios(config) + const data = response.data as HackerNewsSearchResponseRaw + + for (const item of data.hits) { + if (!existing.has(item.objectID)) { + out.push({ + keywords: [keyword], + postId: item.objectID, + }) + existing.add(item.objectID) + } + } + } + return out } catch (err) { logger.error({ err, input }, 'Error while getting posts by keyword in EagleEye') throw err diff --git a/backend/src/services/__tests__/eagleEyeContentService.test.ts b/backend/src/services/__tests__/eagleEyeContentService.test.ts index 46b7d51a8b..9170b4a645 100644 --- a/backend/src/services/__tests__/eagleEyeContentService.test.ts +++ b/backend/src/services/__tests__/eagleEyeContentService.test.ts @@ -1,44 +1,9 @@ -import moment from 'moment' import SequelizeTestUtils from '../../database/utils/sequelizeTestUtils' -import { PlatformType } from '../../types/integrationEnums' +import { EagleEyeActionType, EagleEyeContent } from '../../types/eagleEyeTypes' import EagleEyeContentService from '../eagleEyeContentService' const db = null -const toUpsert = { - keywords: ['keyword'], - similarityScore: 1, - userAttributes: { - [PlatformType.GITHUB]: 'github', - }, - username: 'username', - timestamp: moment().unix(), - url: 'url', - title: 'title', - text: 'text', - platform: 'platform', - sourceId: 'sourceId', - vectorId: '1234', - status: null, -} - -const toUpsert2 = { - keywords: ['keyword'], - similarityScore: 1, - userAttributes: { - [PlatformType.GITHUB]: 'github', - }, - username: 'username', - timestamp: moment().subtract(1, 'days').unix(), - url: 'url', - title: 'title', - text: 'text', - platform: 'platform', - sourceId: 'sourceId2', - vectorId: '12345', - status: null, -} - describe('EagleEyeContentService tests', () => { beforeEach(async () => { await SequelizeTestUtils.wipeDatabase(db) @@ -49,91 +14,54 @@ describe('EagleEyeContentService tests', () => { await SequelizeTestUtils.closeConnection(db) }) - describe('bulk upsert method', () => { - it('Should upsert a single record', async () => { - const mockIRepositoryOptions = await SequelizeTestUtils.getTestIRepositoryOptions(db) - const service = new EagleEyeContentService(mockIRepositoryOptions) - await service.bulkUpsert([toUpsert]) - const result = await service.findAndCountAll({}) - expect(result.count).toBe(1) - }) - - it('Should upsert a single record', async () => { - const mockIRepositoryOptions = await SequelizeTestUtils.getTestIRepositoryOptions(db) - const service = new EagleEyeContentService(mockIRepositoryOptions) - await service.bulkUpsert([toUpsert]) - await service.bulkUpsert([toUpsert2]) - const result = await service.findAndCountAll({}) - expect(result.count).toBe(2) - }) - }) - - describe('findAndCount all method', () => { - it('Should find records', async () => { + describe('upsert method', () => { + it('Should create or update a single content using URL field', async () => { const mockIRepositoryOptions = await SequelizeTestUtils.getTestIRepositoryOptions(db) - const service = new EagleEyeContentService(mockIRepositoryOptions) - await service.bulkUpsert([toUpsert]) - await service.bulkUpsert([toUpsert2]) - const result = await service.findAndCountAll({}) - expect(result.count).toBe(2) - expect(result.rows[1].vectorId).toBe(toUpsert.vectorId) - expect(result.rows[0].vectorId).toBe(toUpsert2.vectorId) - }) + const content: EagleEyeContent = { + platform: 'reddit', + url: 'https://some-post-url', + post: { + title: 'post title', + body: 'post body', + }, + postedAt: '2020-05-27T15:13:30Z', + tenantId: mockIRepositoryOptions.currentTenant.id, + actions: [ + { + type: EagleEyeActionType.BOOKMARK, + timestamp: '2022-06-27T14:13:30Z', + }, + ], + } - it('Should work when no records', async () => { - const mockIRepositoryOptions = await SequelizeTestUtils.getTestIRepositoryOptions(db) const service = new EagleEyeContentService(mockIRepositoryOptions) - const result = await service.findAndCountAll({}) - expect(result.count).toBe(0) - }) - }) + const c1 = await service.upsert(content) - describe('findNotInbox method', () => { - it('4 records: 2 have status null, one is too old. Return 1', async () => { - const mockIRepositoryOptions = await SequelizeTestUtils.getTestIRepositoryOptions(db) - const service = new EagleEyeContentService(mockIRepositoryOptions) + let contents = await service.query({}) - const nInbox1 = { - keywords: ['keyword'], - similarityScore: 1, - userAttributes: { - [PlatformType.GITHUB]: 'github', - }, - username: 'username', - timestamp: moment().unix(), - url: 'url', - title: 'title', - text: 'text', - platform: 'platform', - sourceId: 'p-321', - vectorId: '321', - status: 'rejected', - } + expect(contents.count).toBe(1) + expect(contents.rows).toStrictEqual([c1]) - const nInbox2 = { - keywords: ['keyword'], - similarityScore: 1, - userAttributes: { - [PlatformType.GITHUB]: 'github', + // upsert previous url with some new fields + const contentWithSameUrl: EagleEyeContent = { + platform: 'reddit', + url: 'https://some-post-url', + post: { + title: 'a brand new post title', + body: 'better post body', }, - username: 'username', - timestamp: moment().subtract(31, 'days').unix(), - url: 'url', - title: 'title', - text: 'text', - platform: 'platform', - sourceId: 'p-4321', - vectorId: '4321', - status: 'engaged', + postedAt: '2020-05-27T15:13:30Z', + tenantId: mockIRepositoryOptions.currentTenant.id, } - await service.bulkUpsert([toUpsert, nInbox1, nInbox2, toUpsert2]) - - const result = await service.findNotInbox() + const c1Upserted = await service.upsert(contentWithSameUrl) - expect(result.length).toBe(1) - expect(result[0]).toBe(nInbox1.vectorId) + contents = await service.query({}) + expect(contents.count).toBe(1) + expect(contents.rows).toStrictEqual([c1Upserted]) + expect(c1Upserted.id).toEqual(c1.id) + expect(contents.rows[0].post).toStrictEqual(contentWithSameUrl.post) }) }) }) diff --git a/backend/src/services/eagleEyeActionService.ts b/backend/src/services/eagleEyeActionService.ts new file mode 100644 index 0000000000..75b71c9178 --- /dev/null +++ b/backend/src/services/eagleEyeActionService.ts @@ -0,0 +1,97 @@ +import EagleEyeActionRepository from '../database/repositories/eagleEyeActionRepository' +import EagleEyeContentRepository from '../database/repositories/eagleEyeContentRepository' +import SequelizeRepository from '../database/repositories/sequelizeRepository' +import Error404 from '../errors/Error404' +import { EagleEyeAction, EagleEyeActionType } from '../types/eagleEyeTypes' +import { IServiceOptions } from './IServiceOptions' +import { LoggingBase } from './loggingBase' + +export default class EagleEyeActionService extends LoggingBase { + options: IServiceOptions + + constructor(options) { + super(options) + this.options = options + } + + async create(data: EagleEyeAction, contentId: string): Promise { + const transaction = await SequelizeRepository.createTransaction(this.options) + + // find content + const content = await EagleEyeContentRepository.findById(contentId, this.options) + + if (!content) { + throw new Error404(this.options.language, 'errors.eagleEye.contentNotFound') + } + + const existingUserActions: EagleEyeAction[] = content.actions.filter( + (a) => a.actionById === this.options.currentUser.id, + ) + + const existingUserActionTypes = existingUserActions.map((a) => a.type) + + try { + // check if already bookmarked - if yes ignore the new action and return existing + if ( + data.type === EagleEyeActionType.BOOKMARK && + existingUserActionTypes.includes(EagleEyeActionType.BOOKMARK) + ) { + return existingUserActions.find((a) => a.type === EagleEyeActionType.BOOKMARK) + } + + // thumbs up and down should be mutually exclusive + if ( + data.type === EagleEyeActionType.THUMBS_DOWN && + existingUserActionTypes.includes(EagleEyeActionType.THUMBS_UP) + ) { + await EagleEyeActionRepository.removeActionFromContent( + EagleEyeActionType.THUMBS_UP, + contentId, + this.options, + ) + } else if ( + data.type === EagleEyeActionType.THUMBS_UP && + existingUserActionTypes.includes(EagleEyeActionType.THUMBS_DOWN) + ) { + await EagleEyeActionRepository.removeActionFromContent( + EagleEyeActionType.THUMBS_DOWN, + contentId, + this.options, + ) + } + + // add new action + const record = await EagleEyeActionRepository.createActionForContent( + data, + contentId, + this.options, + ) + + await SequelizeRepository.commitTransaction(transaction) + + return record + } catch (error) { + await SequelizeRepository.rollbackTransaction(transaction) + + SequelizeRepository.handleUniqueFieldError(error, this.options.language, 'EagleEyeContent') + + throw error + } + } + + async destroy(id: string) { + const action = await EagleEyeActionRepository.findById(id, this.options) + + const contentId = action.contentId + + await EagleEyeActionRepository.destroy(id, this.options) + + // find content + const content = await EagleEyeContentRepository.findById(contentId, this.options) + + // if content no longer has any actions attached to it, also delete the content + if (content.actions.length === 0) { + await EagleEyeContentRepository.destroy(contentId, this.options) + } + } +} diff --git a/backend/src/services/eagleEyeContentService.ts b/backend/src/services/eagleEyeContentService.ts index e42de0de9a..3220cef2f1 100644 --- a/backend/src/services/eagleEyeContentService.ts +++ b/backend/src/services/eagleEyeContentService.ts @@ -1,30 +1,25 @@ import moment from 'moment' -import request from 'superagent' -import { API_CONFIG } from '../config' -import SequelizeRepository from '../database/repositories/sequelizeRepository' +import axios from 'axios' +import { EAGLE_EYE_CONFIG } from '../config' import { IServiceOptions } from './IServiceOptions' import EagleEyeContentRepository from '../database/repositories/eagleEyeContentRepository' -import Error400 from '../errors/Error400' -import track from '../segment/track' import { LoggingBase } from './loggingBase' +import { + EagleEyeContent, + EagleEyeAction, + EagleEyeSettings, + EagleEyePublishedDates, + EagleEyeRawPost, + EagleEyePostWithActions, +} from '../types/eagleEyeTypes' +import { PageData, QueryData } from '../types/common' +import Error400 from '../errors/Error400' +import UserRepository from '../database/repositories/userRepository' -interface EagleEyeSearchPoint { - vectorId: string - sourceId: string - title: string - text?: string - url: string - timestamp: number - username: string - similarityScore: number - userAttributes: { - [platform: string]: string - } - keywords: string[] +export interface EagleEyeContentUpsertData extends EagleEyeAction { + content: EagleEyeContent } -type EagleEyeSearchOutput = EagleEyeSearchPoint[] - export default class EagleEyeContentService extends LoggingBase { options: IServiceOptions @@ -33,51 +28,36 @@ export default class EagleEyeContentService extends LoggingBase { this.options = options } - async upsert(data) { - const transaction = await SequelizeRepository.createTransaction(this.options) - - try { - const record = await EagleEyeContentRepository.upsert(data, { - ...this.options, - transaction, - }) - - await SequelizeRepository.commitTransaction(transaction) + /** + * Create an eagle eye shown content record. + * @param data Data to a new EagleEyeContent record. + * @param options Repository options. + * @returns Created EagleEyeContent record. + */ + async upsert(data: EagleEyeContent): Promise { + if (!data.url) { + throw new Error400(this.options.language, 'errors.eagleEye.urlRequiredWhenUpserting') + } - return record - } catch (error) { - await SequelizeRepository.rollbackTransaction(transaction) + // find by url + const existing = await EagleEyeContentRepository.findByUrl(data.url, this.options) - SequelizeRepository.handleUniqueFieldError(error, this.options.language, 'EagleEyeContent') + let record - throw error + if (existing) { + record = await EagleEyeContentRepository.update(existing.id, data, this.options) + } else { + record = await EagleEyeContentRepository.create(data, this.options) } - } - async findNotInbox() { - const shown = ( - await EagleEyeContentRepository.findAndCountAll( - { - filter: { - timestampRange: [ - moment().subtract(30, 'days').toDate(), - moment().add(1, 'hour').toDate(), - ], - status: 'NOT_NULL', - }, - }, - this.options, - ) - ).rows - // Slicing results such that lambda payload will not be too big - return shown.map((record) => record.vectorId).slice(0, 20000) + return record } - async findAndCountAll(args) { - return EagleEyeContentRepository.findAndCountAll(args, this.options) + async findById(id: string): Promise { + return EagleEyeContentRepository.findById(id, this.options) } - async query(data) { + async query(data: QueryData): Promise> { const advancedFilter = data.filter const orderBy = data.orderBy const limit = data.limit @@ -88,99 +68,102 @@ export default class EagleEyeContentService extends LoggingBase { ) } - async bulkUpsert(data: EagleEyeSearchOutput) { - for (const point of data) { - await this.upsert(point) + /** + * Convert a relative string date to a Date. For example, 30 days ago -> 2020-01-01 + * @param date String date. Can be one of EagleEyePublishedDates + * @returns The corresponding Date + */ + static switchDate(date: string, offset = 0) { + let dateMoment + switch (date) { + case EagleEyePublishedDates.LAST_24_HOURS: + dateMoment = moment().subtract(1, 'days') + break + case EagleEyePublishedDates.LAST_7_DAYS: + dateMoment = moment().subtract(7, 'days') + break + case EagleEyePublishedDates.LAST_14_DAYS: + dateMoment = moment().subtract(14, 'days') + break + case EagleEyePublishedDates.LAST_30_DAYS: + dateMoment = moment().subtract(30, 'days') + break + case EagleEyePublishedDates.LAST_90_DAYS: + dateMoment = moment().subtract(90, 'days') + break + default: + return null } + return dateMoment.add(offset, 'days').format('YYYY-MM-DD') } - async search(args) { - const { keywords, nDays, exactKeywords } = args - // We do not want what we have already accepted or rejected - const filters = await this.findNotInbox() - if (API_CONFIG.premiumApiUrl) { - const response = await request - .post(`${API_CONFIG.premiumApiUrl}/search`) - .send({ queries: keywords, nDays, filters, exactKeywords }) - const fromEagleEye: EagleEyeSearchOutput = JSON.parse(response.text) - await this.bulkUpsert(fromEagleEye) - return fromEagleEye + async search(email = false) { + const eagleEyeSettings: EagleEyeSettings = ( + await UserRepository.findById(this.options.currentUser.id, this.options) + ).eagleEyeSettings + + if (!eagleEyeSettings.onboarded) { + throw new Error400(this.options.language, 'errors.eagleEye.notOnboarded') } - return [] as EagleEyeSearchOutput - } - async keywordMatch(args) { - const { keywords, nDays, platform } = args - - if (API_CONFIG.premiumApiUrl) { - const response = await request - .post(`${API_CONFIG.premiumApiUrl}/keyword-match`) - .send({ exactKeywords: keywords, nDays, platform }) - try { - return JSON.parse(response.text) - } catch (error) { - this.log.error({ error: response.error }, 'error while calling eagle eye server!') - throw new Error400('en', 'errors.eagleEyeSearchFailed.message') - } - } else { - return [] as EagleEyeSearchOutput + const feedSettings = email ? eagleEyeSettings.emailDigest.feed : eagleEyeSettings.feed + + const keywords = feedSettings.keywords ? feedSettings.keywords.join(',') : '' + const exactKeywords = feedSettings.exactKeywords ? feedSettings.exactKeywords.join(',') : '' + const excludedKeywords = feedSettings.excludedKeywords + ? feedSettings.excludedKeywords.join(',') + : '' + + const afterDate = EagleEyeContentService.switchDate(feedSettings.publishedDate) + + const config = { + method: 'get', + maxBodyLength: Infinity, + url: `${EAGLE_EYE_CONFIG.url}`, + params: { + platforms: feedSettings.platforms.join(','), + keywords, + exact_keywords: exactKeywords, + exclude_keywords: excludedKeywords, + after_date: afterDate, + }, + headers: { + Authorization: `Bearer ${EAGLE_EYE_CONFIG.apiKey}`, + }, } - } - async update(id, data) { - const transaction = await SequelizeRepository.createTransaction(this.options) + const response = await axios(config) - try { - const recordBeforeUpdate = await EagleEyeContentRepository.findById(id, { ...this.options }) - const record = await EagleEyeContentRepository.update(id, data, { - ...this.options, - transaction, + const interacted = ( + await this.query({ + filter: { + postedAt: { gt: EagleEyeContentService.switchDate(feedSettings.publishedDate, 15) }, + }, }) + ).rows - // If we are updating status we want to track it - if (data.status !== recordBeforeUpdate.status) { - // If we are going from null to status, we are either accepting or rejecting - if (data.status && data.status !== null && data.status !== undefined) { - track( - `EagleEye ${data.status}`, - { - ...data, - platform: record.platform, - keywords: record.keywords, - title: record.title, - url: record.url, - }, - { ...this.options }, - ) - // Here we are bringing back a rejected post to the Inbox - } else if (recordBeforeUpdate.status === 'rejected' && data.status === null) { - track( - `EagleEye post from rejected to Inbox`, - { - ...data, - platform: record.platform, - keywords: record.keywords, - title: record.title, - url: record.url, - }, - { ...this.options }, - ) - } - } - - await SequelizeRepository.commitTransaction(transaction) - - return record - } catch (error) { - await SequelizeRepository.rollbackTransaction(transaction) + const interactedMap = {} - SequelizeRepository.handleUniqueFieldError(error, this.options.language, 'EagleEyeContent') + for (const item of interacted) { + interactedMap[item.url] = item + } - throw error + const out: EagleEyePostWithActions[] = [] + for (const item of response.data as EagleEyeRawPost[]) { + const post = { + description: item.description, + thumbnail: item.thumbnail, + title: item.title, + } + out.push({ + url: item.url, + postedAt: item.date, + post, + platform: item.platform, + actions: interactedMap[item.url] ? interactedMap[item.url].actions : [], + }) } - } - async findById(id) { - return EagleEyeContentRepository.findById(id, this.options) + return out } } diff --git a/backend/src/services/eagleEyeSettingsService.ts b/backend/src/services/eagleEyeSettingsService.ts new file mode 100644 index 0000000000..7753f2f13d --- /dev/null +++ b/backend/src/services/eagleEyeSettingsService.ts @@ -0,0 +1,158 @@ +import lodash from 'lodash' +import SequelizeRepository from '../database/repositories/sequelizeRepository' +import UserRepository from '../database/repositories/userRepository' +import Error400 from '../errors/Error400' +import { + EagleEyeSettings, + EagleEyeFeedSettings, + EagleEyePlatforms, + EagleEyePublishedDates, + EagleEyeEmailDigestSettings, +} from '../types/eagleEyeTypes' +import { IServiceOptions } from './IServiceOptions' +import { LoggingBase } from './loggingBase' + +export default class EagleEyeSettingsService extends LoggingBase { + options: IServiceOptions + + constructor(options) { + super(options) + this.options = options + } + + /** + * Validate and normalize feed settings. + * @param data Feed data of type EagleEyeFeedSettings + * @returns Normalized feed data if the input is valid. Otherwise a 400 Error + */ + getFeed(data: EagleEyeFeedSettings) { + // Feed is compulsory + if (!data) { + throw new Error400(this.options.language, 'errors.eagleEye.feedSettingsMissing') + } + + // We need at least one of keywords or exactKeywords + if (!data.keywords && !data.exactKeywords) { + throw new Error400(this.options.language, 'errors.eagleEye.keywordsMissing') + } + + // We need at least one platform + if (!data.platforms || data.platforms.length === 0) { + throw new Error400(this.options.language, 'errors.eagleEye.platformMissing') + } + + // Make sure platforms are in the allowed list + const platforms = Object.values(EagleEyePlatforms) as string[] + data.platforms.forEach((platform) => { + if (!platforms.includes(platform)) { + throw new Error400( + this.options.language, + 'errors.eagleEye.platformInvalid', + platform, + platforms.join(', '), + ) + } + }) + + // We need a date. Make sure it's in the allowed list. + const publishedDates = Object.values(EagleEyePublishedDates) as string[] + if (publishedDates.indexOf(data.publishedDate) === -1) { + throw new Error400( + this.options.language, + 'errors.eagleEye.publishedDateMissing', + publishedDates.join(', '), + ) + } + + // Remove any extra fields + return lodash.pick(data, [ + 'keywords', + 'exactKeywords', + 'excludedKeywords', + 'publishedDate', + 'platforms', + ]) + } + + /** + * Validate and normalize email digest settings. + * @param data Email digest settings of type EagleEyeEmailDigestSettings + * @param feed Standard feed settings of type EagleEyeFeedSettings + * @returns The normalized email digest settings if the input is valid. Otherwise a 400 Error. + */ + getEmailDigestSettings(data: EagleEyeEmailDigestSettings, feed: EagleEyeFeedSettings) { + // If the matchFeedSettings option is toggled, we set the email feed settings to the standard feed settings. + // Otherwise, we validate and normalize the email feed settings. + if (!data.matchFeedSettings) { + data.feed = this.getFeed(data.feed) + } else { + data.feed = feed + } + + // Make sure the email exists and is valid + const emailRegex = + /^(([^<>()[\]\\.,;:\s@"]+(\.[^<>()[\]\\.,;:\s@"]+)*)|(".+"))@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\])|(([a-zA-Z\-0-9]+\.)+[a-zA-Z]{2,}))$/ + if (!emailRegex.test(data.email)) { + throw new Error400(this.options.language, 'errors.eagleEye.emailInvalid') + } + + // Make sure the frequency exists and is valid + if (['daily', 'weekly'].indexOf(data.frequency) === -1) { + throw new Error400(this.options.language, 'errors.eagleEye.frequencyInvalid') + } + + // Make sure the time exists and is valid + const timeRegex = /^([0-1]?[0-9]|2[0-3]):[0-5][0-9]?$/ + if (!timeRegex.test(data.time)) { + throw new Error400(this.options.language, 'errors.eagleEye.timeInvalid') + } + + // Remove any extra fields + return lodash.pick(data, ['email', 'frequency', 'time', 'matchFeedSettings', 'feed']) + } + + /** + * Validate, normalize and update EagleEye settings. + * @param data Input of type EagleEyeSettings + * @returns Normalized EagleEyeSettings if the input is valid. Otherwise a 400 Error. + */ + async update(data: EagleEyeSettings): Promise { + const transaction = await SequelizeRepository.createTransaction(this.options) + try { + // At this point onboarded is true always + data.onboarded = true + + // Validate and normalize feed settings + data.feed = this.getFeed(data.feed) + + // If an email digest was sent, validate and normalize email digest settings + // Otherwise, set email digest to false + if (data.emailDigest || data.emailDigestActive) { + data.emailDigestActive = true + data.emailDigest = this.getEmailDigestSettings(data.emailDigest, data.feed) + } else { + data.emailDigestActive = false + } + + // Remove any extra fields + data = lodash.pick(data, ['onboarded', 'feed', 'emailDigestActive', 'emailDigest']) + + // Update the user's EagleEye settings + const userOut = await UserRepository.updateEagleEyeSettings( + this.options.currentUser.id, + data, + { ...this.options, transaction }, + ) + + await SequelizeRepository.commitTransaction(transaction) + + return userOut.eagleEyeSettings + } catch (error) { + await SequelizeRepository.rollbackTransaction(transaction) + + SequelizeRepository.handleUniqueFieldError(error, this.options.language, 'EagleEyeContent') + + throw error + } + } +} diff --git a/backend/src/types/common.ts b/backend/src/types/common.ts index 1eb58e2dc9..d7b4b7a253 100644 --- a/backend/src/types/common.ts +++ b/backend/src/types/common.ts @@ -5,6 +5,13 @@ export interface PageData { offset: number } +export interface QueryData { + filter?: any + orderBy?: string + limit?: number + offset?: number +} + export interface SearchCriteria { limit?: number offset?: number diff --git a/backend/src/types/eagleEyeTypes.ts b/backend/src/types/eagleEyeTypes.ts new file mode 100644 index 0000000000..2dccce7703 --- /dev/null +++ b/backend/src/types/eagleEyeTypes.ts @@ -0,0 +1,93 @@ +export enum EagleEyeActionType { + THUMBS_UP = 'thumbs-up', + THUMBS_DOWN = 'thumbs-down', + BOOKMARK = 'bookmark', +} + +export interface EagleEyeAction { + id?: string + type: EagleEyeActionType + timestamp: Date | string + createdAt?: Date | string + updatedAt?: Date | string +} + +export interface EagleEyeContent { + id?: string + platform: string + post: any + url: string + actions?: EagleEyeAction[] + tenantId: string + postedAt: string + createdAt?: Date | string + updatedAt?: Date | string +} + +export interface EagleEyeFeedSettings { + keywords: string[] + exactKeywords: string[] + excludedKeywords: string[] + publishedDate: string + platforms: string[] +} + +export interface EagleEyeEmailDigestSettings { + email: string + frequency: 'daily' | 'weekly' + time: string + feed: EagleEyeFeedSettings + matchFeedSettings: boolean +} + +export interface EagleEyeSettings { + onboarded: boolean + feed: EagleEyeFeedSettings + emailDigestActive: boolean + emailDigest?: EagleEyeEmailDigestSettings +} + +// Enum for EagleEyePlatforms +export enum EagleEyePlatforms { + GITHUB = 'github', + HACKERNEWS = 'hackernews', + DEVTO = 'devto', + REDDIT = 'reddit', + MEDIUM = 'medium', + STACKOVERFLOW = 'stackoverflow', + TWITTER = 'twitter', + YOUTUBE = 'youtube', + PRODUCTHUNT = 'producthunt', + KAGGLE = 'kaggle', + HASHNODE = 'hashnode', + LINKEDIN = 'linkedin', +} + +export enum EagleEyePublishedDates { + LAST_24_HOURS = 'Last 24h', + LAST_7_DAYS = 'Last 7d', + LAST_14_DAYS = 'Last 14d', + LAST_30_DAYS = 'Last 30d', + LAST_90_DAYS = 'Last 90d', +} + +export interface EagleEyeRawPost { + description: string + title: string + thumbnail?: string + url: string + platform: string + date: string +} + +export interface EagleEyePostWithActions { + post: { + description: string + title: string + thumbnail?: string + } + url: string + platform: string + postedAt: string + actions: EagleEyeAction[] +}