From 154603328cdf8d5d1b8091c3b8d6e346fe89bbb5 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Tue, 15 Nov 2022 10:15:16 -0500 Subject: [PATCH 01/11] chore: Exclude jekyll generated files from typescript compiler --- tsconfig.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tsconfig.json b/tsconfig.json index f503bb39..d17213cc 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -18,6 +18,7 @@ "exclude": [ "node_modules", "coverage", - "tests/*.ts" + "tests/*.ts", + "_site" ] } From 718f81b921dde146b23aeb30bcbc115ab14f791b Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Tue, 15 Nov 2022 13:07:40 -0500 Subject: [PATCH 02/11] feat(docs): Generate docs on startup or in docker build * Generate docs if none found on Web client startup * Add "Docs" link to local docs index * Add working implementation of building docs in docker image --- .dockerignore | 6 +- Dockerfile | 10 +++ docs/README.md | 2 +- package-lock.json | 68 +++++++++++++++++++ package.json | 3 + src/Web/Client/index.ts | 83 +++++++++++++++++++++--- src/Web/assets/nodocs.html | 55 ++++++++++++++++ src/Web/assets/public/statusTour.js | 4 +- src/Web/assets/views/partials/header.ejs | 5 ++ src/Web/assets/views/partials/title.ejs | 5 ++ src/util.ts | 17 +++++ 11 files changed, 246 insertions(+), 12 deletions(-) create mode 100644 src/Web/assets/nodocs.html diff --git a/.dockerignore b/.dockerignore index d450481c..167fc0ac 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,7 +1,10 @@ .git logs .github -docs +_site +.bundle +vendor +docs/.jekyll-cache node_modules coverage .nyc_output @@ -23,3 +26,4 @@ coverage !tsconfig.json !package*.json !docker/config/** +!_config.yml diff --git a/Dockerfile b/Dockerfile index 2abee482..1e0ca5bc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -115,6 +115,16 @@ RUN npm install --production \ && rm -rf node_modules/ts-node \ && rm -rf node_modules/typescript +# https://github.com/sass/sassc-ruby/issues/189#issuecomment-629758948 +# sassc is very slow to compile bc there are no alpine binaries +RUN apk add --no-cache --virtual .build-deps \ + make gcc g++ ruby-bundler ruby-dev \ + && gem install bundler:2.3.9 \ + && bundle install \ + && bundle exec jekyll build -b /docs \ + && apk del .build-deps \ + && rm -rf docs + ENV NPM_CONFIG_LOGLEVEL debug # can set database to use more performant better-sqlite3 since we control everything diff --git a/docs/README.md b/docs/README.md index 1817dd62..a74e4878 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,6 +1,6 @@ --- title: Overview -permalink: /overview +permalink: /overview.html nav_order: 2 --- diff --git a/package-lock.json b/package-lock.json index bf71b0fa..658d2f64 100644 --- a/package-lock.json +++ b/package-lock.json @@ -22,6 +22,7 @@ "@nlpjs/lang-fr": "^4.23.4", "@nlpjs/language": "^4.22.7", "@nlpjs/nlp": "^4.23.5", + "@npmcli/promise-spawn": "^6.0.1", "@stdlib/regexp-regexp": "^0.0.6", "ajv": "^7.2.4", "ansi-regex": ">=5.0.1", @@ -30,6 +31,7 @@ "body-parser": "^1.19.0", "cache-manager": "^3.4.4", "cache-manager-redis-store": "^2.0.0", + "command-exists": "^1.2.9", "commander": "^8.0.0", "comment-json": "^4.1.1", "connect-typeorm": "^2.0.0", @@ -98,6 +100,7 @@ "@types/cache-manager-redis-store": "^2.0.0", "@types/chai": "^4.3.0", "@types/chai-as-promised": "^7.1.5", + "@types/command-exists": "^1.2.0", "@types/cookie-parser": "^1.4.2", "@types/express": "^4.17.13", "@types/express-session": "^1.17.4", @@ -979,6 +982,31 @@ "resolved": "https://registry.npmjs.org/@nlpjs/slot/-/slot-4.22.17.tgz", "integrity": "sha512-cNYcxf9DKB+fnRa2NxT5wbWq5j57R1WCTXLWI/1Cyycr227IP7GN7qaD4RbkzotBFFB8wm63UHod9frzmuiXxg==" }, + "node_modules/@npmcli/promise-spawn": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/@npmcli/promise-spawn/-/promise-spawn-6.0.1.tgz", + "integrity": "sha512-+hcUpxgx0vEpDJI9Cn+lkTdKLoqKBXFCVps5H7FujEU2vLOp6KwqjLlxbnz8Wzgm8oEqW/u5FeNAXSFjLdCD0A==", + "dependencies": { + "which": "^3.0.0" + }, + "engines": { + "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + } + }, + "node_modules/@npmcli/promise-spawn/node_modules/which": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/which/-/which-3.0.0.tgz", + "integrity": "sha512-nla//68K9NU6yRiwDY/Q8aU6siKlSs64aEC7+IV56QoAuyQT2ovsJcgGYGyqMOmI/CGN1BOR6mM5EN0FBO+zyQ==", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/which.js" + }, + "engines": { + "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + } + }, "node_modules/@sindresorhus/is": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-4.6.0.tgz", @@ -1199,6 +1227,12 @@ "@types/chai": "*" } }, + "node_modules/@types/command-exists": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@types/command-exists/-/command-exists-1.2.0.tgz", + "integrity": "sha512-ugsxEJfsCuqMLSuCD4PIJkp5Uk2z6TCMRCgYVuhRo5cYQY3+1xXTQkSlPtkpGHuvWMjS2KTeVQXxkXRACMbM6A==", + "dev": true + }, "node_modules/@types/component-emitter": { "version": "1.2.11", "resolved": "https://registry.npmjs.org/@types/component-emitter/-/component-emitter-1.2.11.tgz", @@ -2750,6 +2784,11 @@ "node": ">= 0.8" } }, + "node_modules/command-exists": { + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/command-exists/-/command-exists-1.2.9.tgz", + "integrity": "sha512-LTQ/SGc+s0Xc0Fu5WaKnR0YiygZkm9eKFvyS+fRsU7/ZWFF8ykFM6Pc9aCVf1+xasOOZpO3BAVgVrKvsqKHV7w==" + }, "node_modules/commander": { "version": "8.3.0", "resolved": "https://registry.npmjs.org/commander/-/commander-8.3.0.tgz", @@ -11115,6 +11154,24 @@ "resolved": "https://registry.npmjs.org/@nlpjs/slot/-/slot-4.22.17.tgz", "integrity": "sha512-cNYcxf9DKB+fnRa2NxT5wbWq5j57R1WCTXLWI/1Cyycr227IP7GN7qaD4RbkzotBFFB8wm63UHod9frzmuiXxg==" }, + "@npmcli/promise-spawn": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/@npmcli/promise-spawn/-/promise-spawn-6.0.1.tgz", + "integrity": "sha512-+hcUpxgx0vEpDJI9Cn+lkTdKLoqKBXFCVps5H7FujEU2vLOp6KwqjLlxbnz8Wzgm8oEqW/u5FeNAXSFjLdCD0A==", + "requires": { + "which": "^3.0.0" + }, + "dependencies": { + "which": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/which/-/which-3.0.0.tgz", + "integrity": "sha512-nla//68K9NU6yRiwDY/Q8aU6siKlSs64aEC7+IV56QoAuyQT2ovsJcgGYGyqMOmI/CGN1BOR6mM5EN0FBO+zyQ==", + "requires": { + "isexe": "^2.0.0" + } + } + } + }, "@sindresorhus/is": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-4.6.0.tgz", @@ -11250,6 +11307,12 @@ "@types/chai": "*" } }, + "@types/command-exists": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@types/command-exists/-/command-exists-1.2.0.tgz", + "integrity": "sha512-ugsxEJfsCuqMLSuCD4PIJkp5Uk2z6TCMRCgYVuhRo5cYQY3+1xXTQkSlPtkpGHuvWMjS2KTeVQXxkXRACMbM6A==", + "dev": true + }, "@types/component-emitter": { "version": "1.2.11", "resolved": "https://registry.npmjs.org/@types/component-emitter/-/component-emitter-1.2.11.tgz", @@ -12566,6 +12629,11 @@ "delayed-stream": "~1.0.0" } }, + "command-exists": { + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/command-exists/-/command-exists-1.2.9.tgz", + "integrity": "sha512-LTQ/SGc+s0Xc0Fu5WaKnR0YiygZkm9eKFvyS+fRsU7/ZWFF8ykFM6Pc9aCVf1+xasOOZpO3BAVgVrKvsqKHV7w==" + }, "commander": { "version": "8.3.0", "resolved": "https://registry.npmjs.org/commander/-/commander-8.3.0.tgz", diff --git a/package.json b/package.json index a07d920b..e0ba95ca 100644 --- a/package.json +++ b/package.json @@ -44,6 +44,7 @@ "@nlpjs/lang-fr": "^4.23.4", "@nlpjs/language": "^4.22.7", "@nlpjs/nlp": "^4.23.5", + "@npmcli/promise-spawn": "^6.0.1", "@stdlib/regexp-regexp": "^0.0.6", "ajv": "^7.2.4", "ansi-regex": ">=5.0.1", @@ -52,6 +53,7 @@ "body-parser": "^1.19.0", "cache-manager": "^3.4.4", "cache-manager-redis-store": "^2.0.0", + "command-exists": "^1.2.9", "commander": "^8.0.0", "comment-json": "^4.1.1", "connect-typeorm": "^2.0.0", @@ -120,6 +122,7 @@ "@types/cache-manager-redis-store": "^2.0.0", "@types/chai": "^4.3.0", "@types/chai-as-promised": "^7.1.5", + "@types/command-exists": "^1.2.0", "@types/cookie-parser": "^1.4.2", "@types/express": "^4.17.13", "@types/express-session": "^1.17.4", diff --git a/src/Web/Client/index.ts b/src/Web/Client/index.ts index 0c727908..0633112a 100644 --- a/src/Web/Client/index.ts +++ b/src/Web/Client/index.ts @@ -17,12 +17,28 @@ import { } from "../../Common/interfaces"; import { buildCachePrefix, - defaultFormat, filterLogBySubreddit, filterCriteriaSummary, formatFilterData, - formatLogLineToHtml, filterLogs, getUserAgent, - intersect, isLogLineMinLevel, - LogEntry, parseInstanceLogInfoName, parseInstanceLogName, parseRedditEntity, - parseSubredditLogName, permissions, - randomId, replaceApplicationIdentifier, resultsSummary, sleep, triggeredIndicator, truncateStringToLength + defaultFormat, + filterLogBySubreddit, + filterCriteriaSummary, + formatFilterData, + formatLogLineToHtml, + filterLogs, + getUserAgent, + intersect, + isLogLineMinLevel, + LogEntry, + parseInstanceLogInfoName, + parseInstanceLogName, + parseRedditEntity, + parseSubredditLogName, + permissions, + randomId, + replaceApplicationIdentifier, + resultsSummary, + sleep, + triggeredIndicator, + truncateStringToLength, + fileOrDirectoryExists, resolvePath, fileOrDirectoryIsWriteable } from "../../util"; import {Cache} from "cache-manager"; import session, {Session, SessionData} from "express-session"; @@ -41,6 +57,9 @@ import {arrayMiddle, booleanMiddle} from "../Common/middleware"; import { URL } from "url"; import {MESSAGE} from "triple-beam"; import Autolinker from "autolinker"; +import commandExists from 'command-exists'; +// @ts-ignore +import promiseSpawn from '@npmcli/promise-spawn'; import path from "path"; import {ExtendedSnoowrap} from "../../Utils/SnoowrapClients"; import ClientUser from "../Common/User/ClientUser"; @@ -63,8 +82,8 @@ import { CMInstanceInterface, HeartbeatResponse, InviteData, SubredditInviteDataPersisted } from "../Common/interfaces"; -import {open} from "fs/promises"; -import {createCacheManager} from "../../Common/Cache"; +import {open, rename, mkdir, copyFile} from "fs/promises"; +import {Logger} from "winston"; const emitter = new EventEmitter(); @@ -105,6 +124,8 @@ app.use('/public', express.static(`${__dirname}/../assets/public`, staticOpts)); app.use('/monaco', express.static(`${__dirname}/../../../node_modules/monaco-editor/`, staticOpts)); app.use('/schemas', express.static(`${__dirname}/../../Schema/`, staticOpts)); +app.use('/docs', express.static(`${__dirname}/../../../_site/`, staticOpts)); + app.use((req, res, next) => { // https://developers.google.com/search/docs/advanced/crawling/block-indexing#http-response-header res.setHeader('X-Robots-Tag', 'noindex'); @@ -153,6 +174,50 @@ const createToken = (bot: CMInstanceInterface, user?: Express.User | any, ) => { }); } +const docsSetup = async (parentLogger: Logger) => { + const logger = parentLogger.child({leaf: 'Docs'}); + const docIndexDir = resolvePath('../../../_site/index.html', __dirname); + const placeholderPath = resolvePath('../assets/nodocs.html', __dirname); + const docFullHintIndexDir = resolvePath('../../../_site/favicon.ico', __dirname); + const siteIndexDir = resolvePath('../../../_site', __dirname); + let siteWriteable = false; + try { + if(!(await fileOrDirectoryExists(docFullHintIndexDir))) { + logger.info('Site has not been generated! Will try to do this now.'); + try { + + siteWriteable = fileOrDirectoryIsWriteable(siteIndexDir); + try { + await commandExists('bundle'); + } catch (e) { + throw new SimpleError(`Cannot generate because 'bundle' is not installed.`); + } + try { + const res = await promiseSpawn('bundle', ['exec','jekyll','build', '-b', '/docs'], { + cwd: resolvePath('../../../', __dirname), + }); + logger.debug(res.stdout); + } catch (e) { + throw new ErrorWithCause(`Error occurred while running 'bundle exec jekyll build'`, {cause: e}); + } + logger.info('Docs built!'); + } catch (e) { + // set up placeholder since docs did not exist and were not generated + logger.warn(new ErrorWithCause('Unable to generate docs site', {cause: e})); + if(siteWriteable && !(await fileOrDirectoryExists(docIndexDir))) { + await mkdir(siteIndexDir); + await copyFile(placeholderPath, docIndexDir); + } + } + } else { + logger.verbose('Docs are already generated.'); + } + } catch (e) { + logger.warn('Unable to setup docs', {cause: e}); + return; + } +} + const peekTrunc = truncateStringToLength(200); const availableLevels = ['error', 'warn', 'info', 'verbose', 'debug']; @@ -205,6 +270,8 @@ const webClient = async (options: OperatorConfigWithFileContext) => { const logger = getLogger({defaultLabel: 'Web', ...options.logging}, 'Web'); + await docsSetup(logger); + logger.stream().on('log', (log: LogInfo) => { emitter.emit('log', log); webLogs.unshift(log); diff --git a/src/Web/assets/nodocs.html b/src/Web/assets/nodocs.html new file mode 100644 index 00000000..d1d16d1e --- /dev/null +++ b/src/Web/assets/nodocs.html @@ -0,0 +1,55 @@ + + + + + + + Docs - ContextMod + + + + + + + + + + + + + + + + + +
+
+
+
+
+ + No Docs! +
+
+
+
+
+
+ +
+
+
+
+
No Docs Yet!
+
+
Documentation for ContextMod needs to be generated!
+
Likely you do not have dependencies required to generate documentation. See local development documentation for how to do this.
+
Alternatively, official documentation is available at https://contextmod.dev
+
+
+
+
+
+
+ + diff --git a/src/Web/assets/public/statusTour.js b/src/Web/assets/public/statusTour.js index 8a4a44b1..2dfd5850 100644 --- a/src/Web/assets/public/statusTour.js +++ b/src/Web/assets/public/statusTour.js @@ -6,8 +6,8 @@ steps = [ intro: `
The dashboard allows you to monitor and configure your Bot's behavior for each Subreddit it runs on.
<% } %>
+
+ + Docs + +
<% if(locals.showHelp === true) { %>
diff --git a/src/Web/assets/views/partials/title.ejs b/src/Web/assets/views/partials/title.ejs index 9350ec38..00a8383e 100644 --- a/src/Web/assets/views/partials/title.ejs +++ b/src/Web/assets/views/partials/title.ejs @@ -7,6 +7,11 @@ <%= title %> <% } %>
+
+ + Docs + +
Logout diff --git a/src/util.ts b/src/util.ts index 692a8a1c..6957cc96 100644 --- a/src/util.ts +++ b/src/util.ts @@ -1717,6 +1717,23 @@ export const fileOrDirectoryIsWriteable = (location: string) => { } } +export const fileOrDirectoryExists = async (path: string) => { + const pathInfo = pathUtil.parse(path); + const isDir = pathInfo.ext === ''; + try { + await promises.access(path, constants.R_OK); + return true; + } catch (err: any) { + const {code} = err; + if (code === 'ENOENT') { + return false; + } else if (code === 'EACCES') { + throw new SimpleError(`${isDir ? 'Directory' : 'File'} may exist at ${path} but application does not have permission to write to it.`); + } + throw new ErrorWithCause(`Unable to determine if ${isDir ? 'directory' : 'file'} exists at ${path} because application is unable to access the parent directory due to a system error`, {cause: err}); + } +} + export const overwriteMerge = (destinationArray: any[], sourceArray: any[], options: any): any[] => sourceArray; export const removeUndefinedKeys = >(obj: T): T | undefined => { From 51837472193fe46db94299b551d0c172effc1727 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Tue, 15 Nov 2022 13:38:36 -0500 Subject: [PATCH 03/11] feat(docs): Improve jekyll dependencies to speed up docker image build --- Dockerfile | 25 +++++++++++++++---------- Gemfile | 5 ++++- Gemfile.lock | 28 ++++++++++++---------------- 3 files changed, 31 insertions(+), 27 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1e0ca5bc..34fae608 100644 --- a/Dockerfile +++ b/Dockerfile @@ -96,11 +96,26 @@ WORKDIR /app FROM base as build +# copy NPM dependencies and install COPY --chown=abc:abc package*.json ./ COPY --chown=abc:abc tsconfig.json . RUN npm install +# copy bundle/jekyll dependencies and docs folder +COPY --chown=abc:abc Gemfile Gemfile.lock _config.yml ./ +COPY --chown=abc:abc docs ./docs/ + +# sassc (a jekll dependency) is very slow to compile bc there are no alpine binaries +# https://github.com/sass/sassc-ruby/issues/189#issuecomment-629758948 +# so for now sync used jekyll version with prebuilt binary available in alpine repo +RUN apk add --no-cache --virtual .build-deps \ + ruby-jekyll \ + && bundle install \ + && jekyll build -b /docs \ + && apk del .build-deps \ + && rm -rf docs + COPY --chown=abc:abc . /app RUN npm run build && rm -rf node_modules @@ -115,16 +130,6 @@ RUN npm install --production \ && rm -rf node_modules/ts-node \ && rm -rf node_modules/typescript -# https://github.com/sass/sassc-ruby/issues/189#issuecomment-629758948 -# sassc is very slow to compile bc there are no alpine binaries -RUN apk add --no-cache --virtual .build-deps \ - make gcc g++ ruby-bundler ruby-dev \ - && gem install bundler:2.3.9 \ - && bundle install \ - && bundle exec jekyll build -b /docs \ - && apk del .build-deps \ - && rm -rf docs - ENV NPM_CONFIG_LOGLEVEL debug # can set database to use more performant better-sqlite3 since we control everything diff --git a/Gemfile b/Gemfile index 352d480b..a12a9b91 100644 --- a/Gemfile +++ b/Gemfile @@ -1,6 +1,9 @@ source 'https://rubygems.org' -gem "jekyll", "~> 4.3" # installed by `gem jekyll` +# sassc (a jekll dependency) is very slow to compile bc there are no alpine binaries +# https://github.com/sass/sassc-ruby/issues/189#issuecomment-629758948 +# so for now sync used jekyll version with prebuilt binary available in alpine repo +gem "jekyll", "4.2.2" # installed by `gem jekyll` # gem "webrick" # required when using Ruby >= 3 and Jekyll <= 4.2.2 gem "just-the-docs", "0.4.0.rc3" # currently the latest pre-release diff --git a/Gemfile.lock b/Gemfile.lock index e0bb574f..c0304c4f 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -14,22 +14,21 @@ GEM http_parser.rb (0.8.0) i18n (1.12.0) concurrent-ruby (~> 1.0) - jekyll (4.3.0) + jekyll (4.2.2) addressable (~> 2.4) colorator (~> 1.0) em-websocket (~> 0.5) i18n (~> 1.0) - jekyll-sass-converter (>= 2.0, < 4.0) + jekyll-sass-converter (~> 2.0) jekyll-watch (~> 2.0) - kramdown (~> 2.3, >= 2.3.1) + kramdown (~> 2.3) kramdown-parser-gfm (~> 1.0) liquid (~> 4.0) - mercenary (>= 0.3.6, < 0.5) + mercenary (~> 0.4.0) pathutil (~> 0.9) - rouge (>= 3.0, < 5.0) + rouge (~> 3.0) safe_yaml (~> 1.0) - terminal-table (>= 1.8, < 4.0) - webrick (~> 1.7) + terminal-table (~> 2.0) jekyll-default-layout (0.1.5) jekyll (>= 3.0, < 5.0) jekyll-optional-front-matter (0.3.2) @@ -67,22 +66,19 @@ GEM rb-inotify (0.10.1) ffi (~> 1.0) rexml (3.2.5) - rouge (4.0.0) + rouge (3.30.0) safe_yaml (1.0.5) sassc (2.4.0) ffi (~> 1.9) - terminal-table (3.0.2) - unicode-display_width (>= 1.1.1, < 3) - unicode-display_width (2.3.0) - webrick (1.7.0) + terminal-table (2.0.0) + unicode-display_width (~> 1.1, >= 1.1.1) + unicode-display_width (1.8.0) PLATFORMS - arm64-darwin-21 - x86_64-darwin-19 x86_64-linux DEPENDENCIES - jekyll (~> 4.3) + jekyll (= 4.2.2) jekyll-default-layout jekyll-optional-front-matter jekyll-readme-index @@ -91,4 +87,4 @@ DEPENDENCIES just-the-docs (= 0.4.0.rc3) BUNDLED WITH - 2.3.9 + 2.3.25 From 0a2b13e4c4103c679c2c83de7455c8d8e2922514 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Wed, 16 Nov 2022 10:18:37 -0500 Subject: [PATCH 04/11] fix: Fix including self activities in Recent Activity without filtering * Consolidate ACID check for author history results into authorActivities function so it can be used everywhere * Remove self check in Recent Activity and used consolidated functionality so that filtering still occurs on current activity --- src/Rule/RecentActivityRule.ts | 44 +++-------------------------- src/Subreddit/SubredditResources.ts | 39 +++++++++++++++++-------- 2 files changed, 31 insertions(+), 52 deletions(-) diff --git a/src/Rule/RecentActivityRule.ts b/src/Rule/RecentActivityRule.ts index fb34ca7c..f0ba735e 100644 --- a/src/Rule/RecentActivityRule.ts +++ b/src/Rule/RecentActivityRule.ts @@ -126,28 +126,7 @@ export class RecentActivityRule extends Rule { async process(item: Submission | Comment): Promise<[boolean, RuleResult]> { let activities; - // ACID is a bitch - // reddit may not return the activity being checked in the author's recent history due to availability/consistency issues or *something* - // so make sure we add it in if config is checking the same type and it isn't included - // TODO refactor this for SubredditState everywhere branch - let shouldIncludeSelf = true; const strongWindow = windowConfigToWindowCriteria(this.window); - const { - filterOn: { - post: { - subreddits: { - include = [], - exclude = [] - } = {}, - } = {}, - } = {} - } = strongWindow; - // typeof x === string -- a patch for now...technically this is all it supports but eventually will need to be able to do any SubredditState - if (include.length > 0 && !include.some(x => x.name !== undefined && x.name.toLocaleLowerCase() === item.subreddit.display_name.toLocaleLowerCase())) { - shouldIncludeSelf = false; - } else if (exclude.length > 0 && exclude.some(x => x.name !== undefined && x.name.toLocaleLowerCase() === item.subreddit.display_name.toLocaleLowerCase())) { - shouldIncludeSelf = false; - } if(strongWindow.fetch === undefined && this.lookAt !== undefined) { switch(this.lookAt) { @@ -159,25 +138,10 @@ export class RecentActivityRule extends Rule { } } - activities = await this.resources.getAuthorActivities(item.author, strongWindow); - - switch (strongWindow.fetch) { - case 'comment': - if (shouldIncludeSelf && item instanceof Comment && !activities.some(x => x.name === item.name)) { - activities.unshift(item); - } - break; - case 'submission': - if (shouldIncludeSelf && item instanceof Submission && !activities.some(x => x.name === item.name)) { - activities.unshift(item); - } - break; - default: - if (shouldIncludeSelf && !activities.some(x => x.name === item.name)) { - activities.unshift(item); - } - break; - } + // ACID is a bitch + // reddit may not return the activity being checked in the author's recent history due to availability/consistency issues or *something* + // so add current activity as a prefetched activity and add it to the returned activities (after it goes through filtering) + activities = await this.resources.getAuthorActivities(item.author, strongWindow, undefined, [item]); let viableActivity = activities; // if config does not specify reference then we set the default based on whether the item is a submission or not diff --git a/src/Subreddit/SubredditResources.ts b/src/Subreddit/SubredditResources.ts index 216f99cb..f56ed9fa 100644 --- a/src/Subreddit/SubredditResources.ts +++ b/src/Subreddit/SubredditResources.ts @@ -1030,13 +1030,13 @@ export class SubredditResources { } } - async getAuthorActivities(user: RedditUser, options: ActivityWindowCriteria, customListing?: NamedListing): Promise { + async getAuthorActivities(user: RedditUser, options: ActivityWindowCriteria, customListing?: NamedListing, prefetchedActivities?: SnoowrapActivity[]): Promise { - const {post} = await this.getAuthorActivitiesWithFilter(user, options, customListing); + const {post} = await this.getAuthorActivitiesWithFilter(user, options, customListing, prefetchedActivities); return post; } - async getAuthorActivitiesWithFilter(user: RedditUser, options: ActivityWindowCriteria, customListing?: NamedListing): Promise { + async getAuthorActivitiesWithFilter(user: RedditUser, options: ActivityWindowCriteria, customListing?: NamedListing, prefetchedActivities?: SnoowrapActivity[]): Promise { let listFuncName: string; let listFunc: ListingFunc; @@ -1064,21 +1064,21 @@ export class SubredditResources { ...(cloneDeep(options)), } - return await this.getActivities(user, criteriaWithDefaults, {func: listFunc, name: listFuncName}); + return await this.getActivities(user, criteriaWithDefaults, {func: listFunc, name: listFuncName}, prefetchedActivities); } - async getAuthorComments(user: RedditUser, options: ActivityWindowCriteria): Promise { - return await this.getAuthorActivities(user, {...options, fetch: 'comment'}) as unknown as Promise; + async getAuthorComments(user: RedditUser, options: ActivityWindowCriteria, prefetchedActivities?: SnoowrapActivity[]): Promise { + return await this.getAuthorActivities(user, {...options, fetch: 'comment'}, undefined, prefetchedActivities) as unknown as Promise; } - async getAuthorSubmissions(user: RedditUser, options: ActivityWindowCriteria): Promise { + async getAuthorSubmissions(user: RedditUser, options: ActivityWindowCriteria, prefetchedActivities?: SnoowrapActivity[]): Promise { return await this.getAuthorActivities(user, { ...options, fetch: 'submission' - }) as unknown as Promise; + }, undefined,prefetchedActivities) as unknown as Promise; } - async getActivities(user: RedditUser, options: ActivityWindowCriteria, listingData: NamedListing): Promise { + async getActivities(user: RedditUser, options: ActivityWindowCriteria, listingData: NamedListing, prefetchedActivities: SnoowrapActivity[] = []): Promise { try { @@ -1213,12 +1213,24 @@ export class SubredditResources { } } - let unFilteredItems: SnoowrapActivity[] | undefined; - + let preFilteredPrefetchedActivities = [...prefetchedActivities]; + if(preFilteredPrefetchedActivities.length > 0) { + switch(options.fetch) { + // TODO this may not work if using a custom listingFunc that does not include fetch type + case 'comment': + preFilteredPrefetchedActivities = preFilteredPrefetchedActivities.filter(x => asComment(x)); + break; + case 'submission': + preFilteredPrefetchedActivities = preFilteredPrefetchedActivities.filter(x => asSubmission(x)); + break; + } + preFilteredPrefetchedActivities = await this.filterListingWithHistoryOptions(preFilteredPrefetchedActivities, user, options.filterOn?.pre); + } + let unFilteredItems: SnoowrapActivity[] | undefined = [...preFilteredPrefetchedActivities]; + pre = pre.concat(preFilteredPrefetchedActivities); const { func: listingFunc } = listingData; - let listing = await listingFunc(getAuthorHistoryAPIOptions(options)); let hitEnd = false; let offset = chunkSize; @@ -1228,6 +1240,9 @@ export class SubredditResources { timeOk = false; let listSlice = listing.slice(offset - chunkSize); + // filter out any from slice that were already included from prefetched list so that prefetched aren't included twice + listSlice = preFilteredPrefetchedActivities.length === 0 ? listSlice : listSlice.filter(x => !preFilteredPrefetchedActivities.some(y => y.name === x.name)); + let preListSlice = await this.filterListingWithHistoryOptions(listSlice, user, options.filterOn?.pre); // its more likely the time criteria is going to be hit before the count criteria From 9031f7fec8416f6849b19b072f70d78c993962d9 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Thu, 17 Nov 2022 12:03:55 -0500 Subject: [PATCH 05/11] refactor(polling): Improve resilience for polling source parsing * Replace hard-coded polling sources with string constants * Implement string to PollOn parsing which is case-insensitive and forgives mispelling * Check that manager specifies only one of each polling source type when build config --- src/Bot/index.ts | 52 +++++++++++++++------------ src/Common/Infrastructure/Atomic.ts | 13 +++++++ src/Common/interfaces.ts | 2 +- src/ConfigBuilder.ts | 56 +++++++++++++++-------------- src/Subreddit/Manager.ts | 53 +++++++++++++-------------- src/util.ts | 10 ++++++ 6 files changed, 108 insertions(+), 78 deletions(-) diff --git a/src/Bot/index.ts b/src/Bot/index.ts index 9e6dc8b0..0a671c3a 100644 --- a/src/Bot/index.ts +++ b/src/Bot/index.ts @@ -46,7 +46,13 @@ import {RunStateType} from "../Common/Entities/RunStateType"; import {QueueRunState} from "../Common/Entities/EntityRunState/QueueRunState"; import {EventsRunState} from "../Common/Entities/EntityRunState/EventsRunState"; import {ManagerRunState} from "../Common/Entities/EntityRunState/ManagerRunState"; -import {Invokee, PollOn} from "../Common/Infrastructure/Atomic"; +import { + Invokee, + POLLING_COMMENTS, POLLING_MODQUEUE, + POLLING_SUBMISSIONS, + POLLING_UNMODERATED, + PollOn +} from "../Common/Infrastructure/Atomic"; import {FilterCriteriaDefaults} from "../Common/Infrastructure/Filters/FilterShapes"; import {snooLogWrapper} from "../Utils/loggerFactory"; import {InfluxClient} from "../Common/Influx/InfluxClient"; @@ -558,9 +564,9 @@ class Bot implements BotInstanceFunctions { parseSharedStreams() { - const sharedCommentsSubreddits = !this.sharedStreams.includes('newComm') ? [] : this.subManagers.filter(x => x.isPollingShared('newComm')).map(x => x.subreddit.display_name); + const sharedCommentsSubreddits = !this.sharedStreams.includes(POLLING_COMMENTS) ? [] : this.subManagers.filter(x => x.isPollingShared(POLLING_COMMENTS)).map(x => x.subreddit.display_name); if (sharedCommentsSubreddits.length > 0) { - const stream = this.cacheManager.modStreams.get('newComm'); + const stream = this.cacheManager.modStreams.get(POLLING_COMMENTS); if (stream === undefined || stream.subreddit !== sharedCommentsSubreddits.join('+')) { let processed; if (stream !== undefined) { @@ -580,20 +586,20 @@ class Bot implements BotInstanceFunctions { label: 'Shared Polling' }); // @ts-ignore - defaultCommentStream.on('error', this.createSharedStreamErrorListener('newComm')); - defaultCommentStream.on('listing', this.createSharedStreamListingListener('newComm')); - this.cacheManager.modStreams.set('newComm', defaultCommentStream); + defaultCommentStream.on('error', this.createSharedStreamErrorListener(POLLING_COMMENTS)); + defaultCommentStream.on('listing', this.createSharedStreamListingListener(POLLING_COMMENTS)); + this.cacheManager.modStreams.set(POLLING_COMMENTS, defaultCommentStream); } } else { - const stream = this.cacheManager.modStreams.get('newComm'); + const stream = this.cacheManager.modStreams.get(POLLING_COMMENTS); if (stream !== undefined) { stream.end('Determined no managers are listening on shared stream parsing'); } } - const sharedSubmissionsSubreddits = !this.sharedStreams.includes('newSub') ? [] : this.subManagers.filter(x => x.isPollingShared('newSub')).map(x => x.subreddit.display_name); + const sharedSubmissionsSubreddits = !this.sharedStreams.includes(POLLING_SUBMISSIONS) ? [] : this.subManagers.filter(x => x.isPollingShared(POLLING_SUBMISSIONS)).map(x => x.subreddit.display_name); if (sharedSubmissionsSubreddits.length > 0) { - const stream = this.cacheManager.modStreams.get('newSub'); + const stream = this.cacheManager.modStreams.get(POLLING_SUBMISSIONS); if (stream === undefined || stream.subreddit !== sharedSubmissionsSubreddits.join('+')) { let processed; if (stream !== undefined) { @@ -613,19 +619,19 @@ class Bot implements BotInstanceFunctions { label: 'Shared Polling' }); // @ts-ignore - defaultSubStream.on('error', this.createSharedStreamErrorListener('newSub')); - defaultSubStream.on('listing', this.createSharedStreamListingListener('newSub')); - this.cacheManager.modStreams.set('newSub', defaultSubStream); + defaultSubStream.on('error', this.createSharedStreamErrorListener(POLLING_SUBMISSIONS)); + defaultSubStream.on('listing', this.createSharedStreamListingListener(POLLING_SUBMISSIONS)); + this.cacheManager.modStreams.set(POLLING_SUBMISSIONS, defaultSubStream); } } else { - const stream = this.cacheManager.modStreams.get('newSub'); + const stream = this.cacheManager.modStreams.get(POLLING_SUBMISSIONS); if (stream !== undefined) { stream.end('Determined no managers are listening on shared stream parsing'); } } - const isUnmoderatedShared = !this.sharedStreams.includes('unmoderated') ? false : this.subManagers.some(x => x.isPollingShared('unmoderated')); - const unmoderatedstream = this.cacheManager.modStreams.get('unmoderated'); + const isUnmoderatedShared = !this.sharedStreams.includes(POLLING_UNMODERATED) ? false : this.subManagers.some(x => x.isPollingShared(POLLING_UNMODERATED)); + const unmoderatedstream = this.cacheManager.modStreams.get(POLLING_UNMODERATED); if (isUnmoderatedShared && unmoderatedstream === undefined) { const defaultUnmoderatedStream = new UnmoderatedStream(this.client, { subreddit: 'mod', @@ -634,15 +640,15 @@ class Bot implements BotInstanceFunctions { label: 'Shared Polling' }); // @ts-ignore - defaultUnmoderatedStream.on('error', this.createSharedStreamErrorListener('unmoderated')); - defaultUnmoderatedStream.on('listing', this.createSharedStreamListingListener('unmoderated')); - this.cacheManager.modStreams.set('unmoderated', defaultUnmoderatedStream); + defaultUnmoderatedStream.on('error', this.createSharedStreamErrorListener(POLLING_UNMODERATED)); + defaultUnmoderatedStream.on('listing', this.createSharedStreamListingListener(POLLING_UNMODERATED)); + this.cacheManager.modStreams.set(POLLING_UNMODERATED, defaultUnmoderatedStream); } else if (!isUnmoderatedShared && unmoderatedstream !== undefined) { unmoderatedstream.end('Determined no managers are listening on shared stream parsing'); } - const isModqueueShared = !this.sharedStreams.includes('modqueue') ? false : this.subManagers.some(x => x.isPollingShared('modqueue')); - const modqueuestream = this.cacheManager.modStreams.get('modqueue'); + const isModqueueShared = !this.sharedStreams.includes(POLLING_MODQUEUE) ? false : this.subManagers.some(x => x.isPollingShared(POLLING_MODQUEUE)); + const modqueuestream = this.cacheManager.modStreams.get(POLLING_MODQUEUE); if (isModqueueShared && modqueuestream === undefined) { const defaultModqueueStream = new ModQueueStream(this.client, { subreddit: 'mod', @@ -651,9 +657,9 @@ class Bot implements BotInstanceFunctions { label: 'Shared Polling' }); // @ts-ignore - defaultModqueueStream.on('error', this.createSharedStreamErrorListener('modqueue')); - defaultModqueueStream.on('listing', this.createSharedStreamListingListener('modqueue')); - this.cacheManager.modStreams.set('modqueue', defaultModqueueStream); + defaultModqueueStream.on('error', this.createSharedStreamErrorListener(POLLING_MODQUEUE)); + defaultModqueueStream.on('listing', this.createSharedStreamListingListener(POLLING_MODQUEUE)); + this.cacheManager.modStreams.set(POLLING_MODQUEUE, defaultModqueueStream); } else if (isModqueueShared && modqueuestream !== undefined) { modqueuestream.end('Determined no managers are listening on shared stream parsing'); } diff --git a/src/Common/Infrastructure/Atomic.ts b/src/Common/Infrastructure/Atomic.ts index 5425b582..07adff1d 100644 --- a/src/Common/Infrastructure/Atomic.ts +++ b/src/Common/Infrastructure/Atomic.ts @@ -111,6 +111,19 @@ export interface DurationObject { export type JoinOperands = 'OR' | 'AND'; export type PollOn = 'unmoderated' | 'modqueue' | 'newSub' | 'newComm'; +export const POLLING_UNMODERATED: PollOn = 'unmoderated'; +export const POLLING_MODQUEUE: PollOn = 'modqueue'; +export const POLLING_SUBMISSIONS: PollOn = 'newSub'; +export const POLLING_COMMENTS: PollOn = 'newComm'; +export const pollOnTypes: PollOn[] = [POLLING_UNMODERATED, POLLING_MODQUEUE, POLLING_SUBMISSIONS, POLLING_COMMENTS]; +export const pollOnTypeMapping: Map = new Map([ + ['unmoderated', POLLING_UNMODERATED], + ['modqueue', POLLING_MODQUEUE], + ['newsub', POLLING_SUBMISSIONS], + ['newcomm', POLLING_COMMENTS], + // be nice if user mispelled + ['newcom', POLLING_COMMENTS] +]); export type ModeratorNames = 'self' | 'automod' | 'automoderator' | string; export type Invokee = 'system' | 'user'; export type RunState = 'running' | 'paused' | 'stopped'; diff --git a/src/Common/interfaces.ts b/src/Common/interfaces.ts index 2952478a..879e7155 100644 --- a/src/Common/interfaces.ts +++ b/src/Common/interfaces.ts @@ -372,7 +372,7 @@ export interface PollingOptions extends PollingDefaults { * * after they have been manually approved from modqueue * * */ - pollOn: 'unmoderated' | 'modqueue' | 'newSub' | 'newComm' + pollOn: PollOn } export interface TTLConfig { diff --git a/src/ConfigBuilder.ts b/src/ConfigBuilder.ts index 52cbc58f..4f8c20e1 100644 --- a/src/ConfigBuilder.ts +++ b/src/ConfigBuilder.ts @@ -8,7 +8,7 @@ import { overwriteMerge, parseBool, parseExternalUrl, parseUrlContext, parseWikiContext, randomId, readConfigFile, - removeUndefinedKeys, resolvePathFromEnvWithRelative, toStrongSharingACLConfig + removeUndefinedKeys, resolvePathFromEnvWithRelative, toPollOn, toStrongSharingACLConfig } from "./util"; import Ajv, {Schema} from 'ajv'; @@ -74,8 +74,8 @@ import {ErrorWithCause} from "pony-cause"; import {RunConfigHydratedData, RunConfigData, RunConfigObject} from "./Run"; import {AuthorRuleConfig} from "./Rule/AuthorRule"; import { - CacheProvider, ConfigFormat, ConfigFragmentParseFunc, - PollOn + CacheProvider, ConfigFormat, ConfigFragmentParseFunc, POLLING_MODQUEUE, POLLING_UNMODERATED, + PollOn, pollOnTypes } from "./Common/Infrastructure/Atomic"; import { asFilterOptionsJson, @@ -452,27 +452,31 @@ export class ConfigBuilder { export const buildPollingOptions = (values: (string | PollingOptions)[]): PollingOptionsStrong[] => { let opts: PollingOptionsStrong[] = []; + let rawOpts: PollingOptions; for (const v of values) { if (typeof v === 'string') { - opts.push({ - pollOn: v as PollOn, - interval: DEFAULT_POLLING_INTERVAL, - limit: DEFAULT_POLLING_LIMIT, - }); + rawOpts = {pollOn: v as PollOn}; // maybeee } else { - const { - pollOn: p, - interval = DEFAULT_POLLING_INTERVAL, - limit = DEFAULT_POLLING_LIMIT, - delayUntil, - } = v; - opts.push({ - pollOn: p as PollOn, - interval, - limit, - delayUntil, - }); + rawOpts = v; } + + const { + pollOn: p, + interval = DEFAULT_POLLING_INTERVAL, + limit = DEFAULT_POLLING_LIMIT, + delayUntil, + } = rawOpts; + + const pVal = toPollOn(p); + if (opts.some(x => x.pollOn === pVal)) { + throw new SimpleError(`Polling source ${pVal} cannot appear more than once in polling options`); + } + opts.push({ + pollOn: pVal, + interval, + limit, + delayUntil, + }); } return opts; } @@ -796,7 +800,7 @@ export const parseDefaultBotInstanceFromArgs = (args: any): BotInstanceJsonConfi heartbeatInterval: heartbeat, }, polling: { - shared: sharedMod ? ['unmoderated', 'modqueue'] : undefined, + shared: sharedMod ? [POLLING_UNMODERATED, POLLING_MODQUEUE] : undefined, }, nanny: { softLimit, @@ -908,7 +912,7 @@ export const parseDefaultBotInstanceFromEnv = (): BotInstanceJsonConfig => { heartbeatInterval: process.env.HEARTBEAT !== undefined ? parseInt(process.env.HEARTBEAT) : undefined, }, polling: { - shared: parseBool(process.env.SHARE_MOD) ? ['unmoderated', 'modqueue'] : undefined, + shared: parseBool(process.env.SHARE_MOD) ? [POLLING_UNMODERATED, POLLING_MODQUEUE] : undefined, }, nanny: { softLimit: process.env.SOFT_LIMIT !== undefined ? parseInt(process.env.SOFT_LIMIT) : undefined, @@ -1525,10 +1529,10 @@ export const buildBotConfig = (data: BotInstanceJsonConfig, opConfig: OperatorCo botCache.provider.prefix = buildCachePrefix([botCache.provider.prefix, 'bot', (botName || objectHash.sha1(botCreds))]); } - let realShared = shared === true ? ['unmoderated', 'modqueue', 'newComm', 'newSub'] : shared; + let realShared: PollOn[] = shared === true ? pollOnTypes : shared.map(toPollOn); if (sharedMod === true) { - realShared.push('unmoderated'); - realShared.push('modqueue'); + realShared.push(POLLING_UNMODERATED); + realShared.push(POLLING_MODQUEUE); } const botLevelStatDefaults = {...statDefaultsFromOp, ...databaseStatisticsDefaults}; @@ -1566,7 +1570,7 @@ export const buildBotConfig = (data: BotInstanceJsonConfig, opConfig: OperatorCo caching: botCache, userAgent, polling: { - shared: [...new Set(realShared)] as PollOn[], + shared: Array.from(new Set(realShared)), stagger, limit, interval, diff --git a/src/Subreddit/Manager.ts b/src/Subreddit/Manager.ts index 77b6e6f6..2e4649f7 100644 --- a/src/Subreddit/Manager.ts +++ b/src/Subreddit/Manager.ts @@ -93,8 +93,8 @@ import {EntityRunState} from "../Common/Entities/EntityRunState/EntityRunState"; import { ActivitySourceValue, EventRetentionPolicyRange, - Invokee, - PollOn, + Invokee, POLLING_COMMENTS, POLLING_MODQUEUE, POLLING_SUBMISSIONS, POLLING_UNMODERATED, + PollOn, pollOnTypes, recordOutputTypes, RunState } from "../Common/Infrastructure/Atomic"; @@ -635,7 +635,7 @@ export class Manager extends EventEmitter implements RunningStates { const configBuilder = new ConfigBuilder({logger: this.logger}); const validJson = configBuilder.validateJson(configObj); const { - polling = [{pollOn: 'unmoderated', limit: DEFAULT_POLLING_LIMIT, interval: DEFAULT_POLLING_INTERVAL}], + polling = [{pollOn: POLLING_SUBMISSIONS, limit: DEFAULT_POLLING_LIMIT, interval: DEFAULT_POLLING_INTERVAL}], caching, credentials, dryRun, @@ -957,7 +957,7 @@ export class Manager extends EventEmitter implements RunningStates { await this.resources.setActivityLastSeenDate(item.name); // if modqueue is running then we know we are checking for new reports every X seconds - if(options.activitySource.identifier === 'modqueue') { + if(options.activitySource.identifier === POLLING_MODQUEUE) { // if the activity is from modqueue and only has one report then we know that report was just created if(item.num_reports === 1 // otherwise if it has more than one report AND we have seen it (its only seen if it has already been stored (in below block)) @@ -1325,25 +1325,20 @@ export class Manager extends EventEmitter implements RunningStates { } } - isPollingShared(streamName: string): boolean { + isPollingShared(streamName: PollOn): boolean { const pollOption = this.pollOptions.find(x => x.pollOn === streamName); - return pollOption !== undefined && pollOption.limit === DEFAULT_POLLING_LIMIT && pollOption.interval === DEFAULT_POLLING_INTERVAL && this.sharedStreams.includes(streamName as PollOn); + return pollOption !== undefined && pollOption.limit === DEFAULT_POLLING_LIMIT && pollOption.interval === DEFAULT_POLLING_INTERVAL && this.sharedStreams.includes(streamName); } async buildPolling() { - const sources: PollOn[] = ['unmoderated', 'modqueue', 'newComm', 'newSub']; + const sources = [...pollOnTypes]; const subName = this.subreddit.display_name; for (const source of sources) { - if (!sources.includes(source)) { - this.logger.error(`'${source}' is not a valid polling source. Valid sources: unmoderated | modqueue | newComm | newSub`); - continue; - } - - const pollOpt = this.pollOptions.find(x => x.pollOn.toLowerCase() === source.toLowerCase()); + const pollOpt = this.pollOptions.find(x => x.pollOn === source); if (pollOpt === undefined) { if(this.sharedStreamCallbacks.has(source)) { this.logger.debug(`Removing listener for shared polling on ${source.toUpperCase()} because it no longer exists in config`); @@ -1366,11 +1361,11 @@ export class Manager extends EventEmitter implements RunningStates { let modStreamType: string | undefined; switch (source) { - case 'unmoderated': + case POLLING_UNMODERATED: if (limit === DEFAULT_POLLING_LIMIT && interval === DEFAULT_POLLING_INTERVAL && this.sharedStreams.includes(source)) { - modStreamType = 'unmoderated'; + modStreamType = POLLING_UNMODERATED; // use default mod stream from resources - stream = this.cacheManager.modStreams.get('unmoderated') as SPoll; + stream = this.cacheManager.modStreams.get(POLLING_UNMODERATED) as SPoll; } else { stream = new UnmoderatedStream(this.client, { subreddit: this.subreddit.display_name, @@ -1380,11 +1375,11 @@ export class Manager extends EventEmitter implements RunningStates { }); } break; - case 'modqueue': + case POLLING_MODQUEUE: if (limit === DEFAULT_POLLING_LIMIT && interval === DEFAULT_POLLING_INTERVAL && this.sharedStreams.includes(source)) { - modStreamType = 'modqueue'; + modStreamType = POLLING_MODQUEUE; // use default mod stream from resources - stream = this.cacheManager.modStreams.get('modqueue') as SPoll; + stream = this.cacheManager.modStreams.get(POLLING_MODQUEUE) as SPoll; } else { stream = new ModQueueStream(this.client, { subreddit: this.subreddit.display_name, @@ -1394,11 +1389,11 @@ export class Manager extends EventEmitter implements RunningStates { }); } break; - case 'newSub': + case POLLING_SUBMISSIONS: if (limit === DEFAULT_POLLING_LIMIT && interval === DEFAULT_POLLING_INTERVAL && this.sharedStreams.includes(source)) { - modStreamType = 'newSub'; + modStreamType = POLLING_SUBMISSIONS; // use default mod stream from resources - stream = this.cacheManager.modStreams.get('newSub') as SPoll; + stream = this.cacheManager.modStreams.get(POLLING_SUBMISSIONS) as SPoll; } else { stream = new SubmissionStream(this.client, { subreddit: this.subreddit.display_name, @@ -1408,11 +1403,11 @@ export class Manager extends EventEmitter implements RunningStates { }); } break; - case 'newComm': + case POLLING_COMMENTS: if (limit === DEFAULT_POLLING_LIMIT && interval === DEFAULT_POLLING_INTERVAL && this.sharedStreams.includes(source)) { - modStreamType = 'newComm'; + modStreamType = POLLING_COMMENTS; // use default mod stream from resources - stream = this.cacheManager.modStreams.get('newComm') as SPoll; + stream = this.cacheManager.modStreams.get(POLLING_COMMENTS) as SPoll; } else { stream = new CommentStream(this.client, { subreddit: this.subreddit.display_name, @@ -1422,6 +1417,8 @@ export class Manager extends EventEmitter implements RunningStates { }); } break; + default: + throw new CMError(`This shouldn't happen! All polling sources are enumerated in switch. Source value: ${source}`) } if (stream === undefined) { @@ -1514,10 +1511,10 @@ export class Manager extends EventEmitter implements RunningStates { } noChecksWarning = (source: PollOn) => (listing: any) => { - if (this.commentChecks.length === 0 && ['modqueue', 'newComm'].some(x => x === source)) { + if (this.commentChecks.length === 0 && [POLLING_MODQUEUE, POLLING_COMMENTS].some(x => x === source)) { this.logger.warn(`Polling '${source.toUpperCase()}' may return Comments but no comments checks were configured.`); } - if (this.submissionChecks.length === 0 && ['unmoderated', 'modqueue', 'newSub'].some(x => x === source)) { + if (this.submissionChecks.length === 0 && [POLLING_UNMODERATED, POLLING_MODQUEUE, POLLING_SUBMISSIONS].some(x => x === source)) { this.logger.warn(`Polling '${source.toUpperCase()}' may return Submissions but no submission checks were configured.`); } } @@ -1670,7 +1667,7 @@ export class Manager extends EventEmitter implements RunningStates { } this.startedAt = dayjs(); - const modQueuePollOpts = this.pollOptions.find(x => x.pollOn === 'modqueue'); + const modQueuePollOpts = this.pollOptions.find(x => x.pollOn === POLLING_MODQUEUE); if(modQueuePollOpts !== undefined) { this.modqueueInterval = modQueuePollOpts.interval; } diff --git a/src/util.ts b/src/util.ts index 6957cc96..bed0a22b 100644 --- a/src/util.ts +++ b/src/util.ts @@ -77,6 +77,7 @@ import { ImageHashCacheData, ModUserNoteLabel, modUserNoteLabels, + PollOn, pollOnTypeMapping, pollOnTypes, RedditEntity, RedditEntityType, RelativeDateTimeMatch, @@ -3088,3 +3089,12 @@ export const toStrongSharingACLConfig = (data: SharingACLConfig | string[]): Str exclude: (data.exclude ?? []).map(x => parseStringToRegexOrLiteralSearch(x)) } } + +export const toPollOn = (val: string | PollOn): PollOn => { + const clean = val.toLowerCase().trim(); + const pVal = pollOnTypeMapping.get(clean); + if(pVal !== undefined) { + return pVal; + } + throw new SimpleError(`'${val}' is not a valid polling source. Valid sources: ${pollOnTypes.join(' | ')}`); +} From d90e88360dbe8f0473a47ed331ee4623b0df613d Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Thu, 17 Nov 2022 13:16:10 -0500 Subject: [PATCH 06/11] feat: Add some author properties for templating --- .../actionTemplating.md | 23 +++++++++++++ src/Subreddit/SubredditResources.ts | 1 + src/Utils/SnoowrapUtils.ts | 33 +++++++++++++++++-- 3 files changed, 55 insertions(+), 2 deletions(-) diff --git a/docs/subreddit-configuration/actionTemplating.md b/docs/subreddit-configuration/actionTemplating.md index 2537f7fe..b9e1bb9b 100644 --- a/docs/subreddit-configuration/actionTemplating.md +++ b/docs/subreddit-configuration/actionTemplating.md @@ -57,6 +57,29 @@ All Actions with `content` have access to this data: | `title` | As comments => the body of the comment. As Submission => title | Test post please ignore | | `shortTitle` | The same as `title` but truncated to 15 characters | test post pleas... | +#### Common Author + +Additionally, `author` has these properties accessible: + +| Name | Description | Example | +|----------------|-------------------------------------|----------| +| `age` | (Approximate) Age of account | 3 months | +| `linkKarma` | Amount of link karma | 10 | +| `commentKarma` | Amount of comment karma | 3 | +| `totalKarma` | Combined link+comment karma | 13 | +| `verified` | Does account have a verified email? | true | + +NOTE: Accessing these properties may require an additional API call so use sparingly on high-volume comments + +##### Example Usage + +``` +The user {{item.author}} has been a redditor for {{item.author.age}} +``` +Produces: + +> The user FoxxMD has been a redditor for 3 months + ### Submissions If the **Activity** is a Submission these additional properties are accessible: diff --git a/src/Subreddit/SubredditResources.ts b/src/Subreddit/SubredditResources.ts index f56ed9fa..97288688 100644 --- a/src/Subreddit/SubredditResources.ts +++ b/src/Subreddit/SubredditResources.ts @@ -1517,6 +1517,7 @@ export class SubredditResources { usernotes, ruleResults, actionResults, + author: (val) => this.getAuthor(val) }); } diff --git a/src/Utils/SnoowrapUtils.ts b/src/Utils/SnoowrapUtils.ts index e20104a4..e3e4e165 100644 --- a/src/Utils/SnoowrapUtils.ts +++ b/src/Utils/SnoowrapUtils.ts @@ -133,6 +133,7 @@ export interface TemplateContext { ruleResults?: RuleResultEntity[] actionResults?: ActionResultEntity[] activity?: SnoowrapActivity + author?: (val: string | RedditUser) => Promise [key: string]: any } @@ -140,11 +141,25 @@ export const renderContent = async (template: string, data: TemplateContext = {} const { usernotes, ruleResults, + author, actionResults, activity, ...restContext } = data; + let fetchedUser: RedditUser | undefined; + // @ts-ignore + const user = async (): Promise => { + if(fetchedUser === undefined) { + if(author !== undefined) { + // @ts-ignore + fetchedUser = await author(activity.author); + } + } + // @ts-ignore + return fetchedUser; + } + let view: GenericContentTemplateData = { botLink: BOT_LINK, ...restContext @@ -171,10 +186,24 @@ export const renderContent = async (template: string, data: TemplateContext = {} view.modmailLink = `https://www.reddit.com/message/compose?to=%2Fr%2F${subreddit}&message=${encodeURIComponent(permalink)}`; + const author: any = { + toString: () => getActivityAuthorName(activity.author) + }; + + if(template.includes('{{item.author.')) { + // @ts-ignore + const auth = await user(); + + author.age = dayjs.unix(auth.created).fromNow(true); + author.linkKarma = auth.link_karma; + author.commentKarma = auth.comment_karma; + author.totalKarma = auth.comment_karma + auth.link_karma; + author.verified = auth.has_verified_email; + } + const templateData: any = { kind: activity instanceof Submission ? 'submission' : 'comment', - // @ts-ignore - author: getActivityAuthorName(await activity.author), + author, votes: activity.score, age: dayjs.duration(dayjs().diff(dayjs.unix(activity.created))).humanize(), permalink, From b094b72d4ada05f57b87847f8d32a33c800365c9 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Mon, 21 Nov 2022 11:29:14 -0500 Subject: [PATCH 07/11] docs: Update docker compose instructions * Specify docker-compose minimum version * Change commands to use new syntax --- docker-compose.yml | 3 +++ docs/operator/installation.md | 6 ++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 5f999e24..07b9dae0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -24,6 +24,9 @@ services: cache: image: 'redis:7-alpine' + volumes: + # on linux will need to make sure this directory has correct permissions for container to access + - './data/cache:/data' database: image: 'mariadb:10.9.3' diff --git a/docs/operator/installation.md b/docs/operator/installation.md index a1857d47..ff1c6151 100644 --- a/docs/operator/installation.md +++ b/docs/operator/installation.md @@ -55,6 +55,8 @@ The included [`docker-compose.yml`](/docker-compose.yml) provides production-rea #### Setup +The included `docker-compose.yml` file is written for **Docker Compose v2.** + For new installations copy [`config.yaml`](/docker/config/docker-compose/config.yaml) into a folder named `data` in the same folder `docker-compose.yml` will be run from. For users migrating their existing CM instances to docker-compose, copy your existing `config.yaml` into the same `data` folder. Read through the comments in both `docker-compose.yml` and `config.yaml` and makes changes to any relevant settings (passwords, usernames, etc...). Ensure that any settings used in both files (EX mariaDB passwords) match. @@ -62,13 +64,13 @@ Read through the comments in both `docker-compose.yml` and `config.yaml` and mak To build and start CM: ```bash -docker-compose up -d +docker compose up -d ``` To include Grafana/Influx dependencies run: ```bash -docker-compose --profile full up -d +docker compose --profile full up -d ``` ## Locally From e79779d980d21c8b003ee86dfc37862d50458cfb Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Mon, 21 Nov 2022 11:43:34 -0500 Subject: [PATCH 08/11] feat: Implement templating for flair actions --- src/Action/SubmissionAction/FlairAction.ts | 23 +++++++++++----------- src/Action/UserFlairAction.ts | 18 ++++++++++------- 2 files changed, 22 insertions(+), 19 deletions(-) diff --git a/src/Action/SubmissionAction/FlairAction.ts b/src/Action/SubmissionAction/FlairAction.ts index a3ef1dd0..8c23c241 100644 --- a/src/Action/SubmissionAction/FlairAction.ts +++ b/src/Action/SubmissionAction/FlairAction.ts @@ -30,15 +30,14 @@ export class FlairAction extends Action { async process(item: Comment | Submission, ruleResults: RuleResultEntity[], actionResults: ActionResultEntity[], options: runCheckOptions): Promise { const dryRun = this.getRuntimeAwareDryrun(options); let flairParts = []; - if(this.text !== '') { - flairParts.push(`Text: ${this.text}`); - } - if(this.css !== '') { - flairParts.push(`CSS: ${this.css}`); - } - if(this.flair_template_id !== '') { - flairParts.push(`Template: ${this.flair_template_id}`); - } + const renderedText = this.text === '' ? '' : await this.renderContent(this.text, item, ruleResults, actionResults) as string; + flairParts.push(`Text: ${renderedText === '' ? '(None)' : renderedText}`); + + const renderedCss = this.css === '' ? '' : await this.renderContent(this.css, item, ruleResults, actionResults) as string; + flairParts.push(`CSS: ${renderedCss === '' ? '(None)' : renderedCss}`); + + flairParts.push(`Template: ${this.flair_template_id === '' ? '(None)' : this.flair_template_id}`); + const flairSummary = flairParts.length === 0 ? 'No flair (unflaired)' : flairParts.join(' | '); this.logger.verbose(flairSummary); if (item instanceof Submission) { @@ -51,9 +50,9 @@ export class FlairAction extends Action { await item.assignFlair({flair_template_id: this.flair_template_id}).then(() => {}); item.link_flair_template_id = this.flair_template_id; } else { - await item.assignFlair({text: this.text, cssClass: this.css}).then(() => {}); - item.link_flair_css_class = this.css; - item.link_flair_text = this.text; + await item.assignFlair({text: renderedText, cssClass: renderedCss}).then(() => {}); + item.link_flair_css_class = renderedCss; + item.link_flair_text = renderedText; } await this.resources.resetCacheForItem(item); } diff --git a/src/Action/UserFlairAction.ts b/src/Action/UserFlairAction.ts index 693fe2bd..4d42edd4 100644 --- a/src/Action/UserFlairAction.ts +++ b/src/Action/UserFlairAction.ts @@ -26,6 +26,8 @@ export class UserFlairAction extends Action { async process(item: Comment | Submission, ruleResults: RuleResultEntity[], actionResults: ActionResultEntity[], options: runCheckOptions): Promise { const dryRun = this.getRuntimeAwareDryrun(options); let flairParts = []; + let renderedText: string | undefined = undefined; + let renderedCss: string | undefined = undefined; if (this.flair_template_id !== undefined) { flairParts.push(`Flair template ID: ${this.flair_template_id}`) @@ -34,10 +36,12 @@ export class UserFlairAction extends Action { } } else { if (this.text !== undefined) { - flairParts.push(`Text: ${this.text}`); + renderedText = await this.renderContent(this.text, item, ruleResults, actionResults) as string; + flairParts.push(`Text: ${renderedText}`); } if (this.css !== undefined) { - flairParts.push(`CSS: ${this.css}`); + renderedCss = await this.renderContent(this.css, item, ruleResults, actionResults) as string; + flairParts.push(`CSS: ${renderedCss}`); } } @@ -58,7 +62,7 @@ export class UserFlairAction extends Action { this.logger.error('Either the flair template ID is incorrect or you do not have permission to access it.'); throw err; } - } else if (this.text === undefined && this.css === undefined) { + } else if (renderedText === undefined && renderedCss === undefined) { // @ts-ignore await item.subreddit.deleteUserFlair(item.author.name); item.author_flair_css_class = null; @@ -68,11 +72,11 @@ export class UserFlairAction extends Action { // @ts-ignore await item.author.assignFlair({ subredditName: item.subreddit.display_name, - cssClass: this.css, - text: this.text, + cssClass: renderedCss, + text: renderedText, }); - item.author_flair_text = this.text ?? null; - item.author_flair_css_class = this.css ?? null; + item.author_flair_text = renderedText ?? null; + item.author_flair_css_class = renderedCss ?? null; } await this.resources.resetCacheForItem(item); if(typeof item.author !== 'string') { From 5bd38d367ad34a78bbed0c6b04eb3047ed832720 Mon Sep 17 00:00:00 2001 From: Marcin Macinski Date: Fri, 25 Nov 2022 17:13:49 +0100 Subject: [PATCH 09/11] assignFlair doesn't work with flair_template_id --- src/Action/SubmissionAction/FlairAction.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Action/SubmissionAction/FlairAction.ts b/src/Action/SubmissionAction/FlairAction.ts index 8c23c241..58664160 100644 --- a/src/Action/SubmissionAction/FlairAction.ts +++ b/src/Action/SubmissionAction/FlairAction.ts @@ -47,7 +47,7 @@ export class FlairAction extends Action { // assignFlair uses /api/flair (mod endpoint) // selectFlair uses /api/selectflair (self endpoint for user to choose their own flair for submission) // @ts-ignore - await item.assignFlair({flair_template_id: this.flair_template_id}).then(() => {}); + await item.selectFlair({flair_template_id: this.flair_template_id}).then(() => {}); item.link_flair_template_id = this.flair_template_id; } else { await item.assignFlair({text: renderedText, cssClass: renderedCss}).then(() => {}); From fde28362087937016613cdc87e2ef06772a2aa3a Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Mon, 28 Nov 2022 14:36:33 -0500 Subject: [PATCH 10/11] chore: remove comments about wrong endpoints At some point maybe this was fixed by reddit silently? --- src/Action/SubmissionAction/FlairAction.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/Action/SubmissionAction/FlairAction.ts b/src/Action/SubmissionAction/FlairAction.ts index 58664160..397d2b75 100644 --- a/src/Action/SubmissionAction/FlairAction.ts +++ b/src/Action/SubmissionAction/FlairAction.ts @@ -43,10 +43,6 @@ export class FlairAction extends Action { if (item instanceof Submission) { if(!this.dryRun) { if (this.flair_template_id) { - // typings are wrong for this function, flair_template_id should be accepted - // assignFlair uses /api/flair (mod endpoint) - // selectFlair uses /api/selectflair (self endpoint for user to choose their own flair for submission) - // @ts-ignore await item.selectFlair({flair_template_id: this.flair_template_id}).then(() => {}); item.link_flair_template_id = this.flair_template_id; } else { From ef372e531e870e3536c2d6193877984995a74e21 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Tue, 29 Nov 2022 09:47:55 -0500 Subject: [PATCH 11/11] fix(database): Prevent usage of LIMIT in session storage driver when db backend is mysql/mariadb Related to freshgiammi-lab/connect-typeorm#8 Closes #128 --- src/Web/Client/StorageProvider.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/Web/Client/StorageProvider.ts b/src/Web/Client/StorageProvider.ts index 81effffc..94d9e574 100644 --- a/src/Web/Client/StorageProvider.ts +++ b/src/Web/Client/StorageProvider.ts @@ -12,6 +12,7 @@ import {Logger} from "winston"; import {WebSetting} from "../../Common/WebEntities/WebSetting"; import {ErrorWithCause} from "pony-cause"; import {createCacheManager} from "../../Common/Cache"; +import {MysqlDriver} from "typeorm/driver/mysql/MysqlDriver"; export interface CacheManagerStoreOptions { prefix?: string @@ -103,7 +104,12 @@ export class DatabaseStorageProvider extends StorageProvider { } createSessionStore(options?: TypeormStoreOptions): Store { - return new TypeormStore(options).connect(this.clientSessionRepo) + // https://github.com/freshgiammi-lab/connect-typeorm#implement-the-session-entity + // https://github.com/freshgiammi-lab/connect-typeorm/issues/8 + // usage of LIMIT in subquery is not supported by mariadb/mysql + // limitSubquery: false -- turns off LIMIT usage + const realOptions = this.database.driver instanceof MysqlDriver ? {...options, limitSubquery: false} : options; + return new TypeormStore(realOptions).connect(this.clientSessionRepo) } async getSessionSecret(): Promise {