From 703a52e118a1a8dfa8b494a70c93420571377838 Mon Sep 17 00:00:00 2001 From: Zakaria Mansouri Date: Sun, 7 Aug 2022 19:27:35 +0200 Subject: [PATCH 1/3] Added robot.txt to disallow indexing non production version of the website --- .github/workflows/cd.deploy.stg.yml | 1 + .github/workflows/cd.deploy.yml | 1 + web/package.json | 1 + web/webpack.config.ts | 15 ++++++++++ yarn.lock | 44 ++++++++++++++++++++++++++++- 5 files changed, 61 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cd.deploy.stg.yml b/.github/workflows/cd.deploy.stg.yml index da7060aa0..4beef5680 100644 --- a/.github/workflows/cd.deploy.stg.yml +++ b/.github/workflows/cd.deploy.stg.yml @@ -63,6 +63,7 @@ jobs: env: CI: true + STAGE: "staging" SSH_KNOWN_HOSTS: ${{ secrets.SSH_KNOWN_HOSTS }} SSH_ADDRESS_STG: ${{ secrets.SSH_ADDRESS_STG }} FIREBASE_TOKEN: ${{ secrets.FIREBASE_TOKEN }} diff --git a/.github/workflows/cd.deploy.yml b/.github/workflows/cd.deploy.yml index c01ca0465..ab0ea2a2f 100644 --- a/.github/workflows/cd.deploy.yml +++ b/.github/workflows/cd.deploy.yml @@ -70,6 +70,7 @@ jobs: env: CI: true + STAGE: "production" SSH_KNOWN_HOSTS: ${{ secrets.SSH_KNOWN_HOSTS }} SSH_ADDRESS_PRD: ${{ secrets.SSH_ADDRESS_PRD }} FIREBASE_TOKEN: ${{ secrets.FIREBASE_TOKEN }} diff --git a/web/package.json b/web/package.json index d96d32612..28dd3af63 100644 --- a/web/package.json +++ b/web/package.json @@ -82,6 +82,7 @@ "postcss-preset-env": "^7.0.1", "pug": "^3.0.0", "pug-loader": "^2.4.0", + "robotstxt-webpack-plugin": "^7.0.0", "sass": "^1.30.0", "sass-loader": "^10.1.0", "sitemap": "^6.3.5", diff --git a/web/webpack.config.ts b/web/webpack.config.ts index 0bcabb46d..867b3c7bf 100644 --- a/web/webpack.config.ts +++ b/web/webpack.config.ts @@ -1,4 +1,5 @@ import { fsConfig } from "@dzcode.io/utils/dist/config"; +import { Environment } from "@dzcode.io/utils/dist/config/environment"; import CssMinimizerPlugin from "css-minimizer-webpack-plugin"; import { readFileSync } from "fs"; import glob from "glob"; @@ -10,9 +11,14 @@ import { Configuration as WPC } from "webpack"; import { BundleAnalyzerPlugin } from "webpack-bundle-analyzer"; import { Configuration as WPDSC } from "webpack-dev-server"; +const RobotstxtPlugin = require("robotstxt-webpack-plugin"); // eslint-disable-line @typescript-eslint/no-var-requires + +// @TODO-ZM: to combine isProduction and isDevelopment and stage + // setting up project configurations and some env variables const ANALYZE = process.env.ANALYZE === "true"; const isProduction = process.env.NODE_ENV === "production" || ANALYZE; +const stage = (process.env.STAGE as Environment) || "production"; const isDevelopment = process.env.NODE_ENV === "development" && !ANALYZE; const { web } = fsConfig("development"); const distFolder = "./bundle"; @@ -157,6 +163,15 @@ export default { new BundleAnalyzerPlugin(), ] : []), + new RobotstxtPlugin({ + policy: [ + { + userAgent: "*", + allow: stage === "production" ? "/" : undefined, + disallow: stage !== "production" ? "/" : undefined, + }, + ], + }), ...apps.reduce( (pV, app) => [...pV, ...require(`./src/apps/${app}/entry/webpack.plugins`)], [], diff --git a/yarn.lock b/yarn.lock index 23ce7ec04..98f9b8dcc 100644 --- a/yarn.lock +++ b/yarn.lock @@ -10624,6 +10624,18 @@ gcp-metadata@^4.2.0: gaxios "^4.0.0" json-bigint "^1.0.0" +generate-robotstxt@^8.0.0: + version "8.0.3" + resolved "https://registry.yarnpkg.com/generate-robotstxt/-/generate-robotstxt-8.0.3.tgz#83384090406e760e9d02247412b2b704562c9b58" + integrity sha512-iD//oAVKcHOCz9M0IiT3pyUiF2uN1qvL3qaTA8RGLz7NU7l0XVwyzd3rN+tzhB657DNUgrygXt9w8+0zkTMFrg== + dependencies: + cosmiconfig "^6.0.0" + fs-extra "^9.0.0" + ip-regex "^4.1.0" + is-absolute-url "^3.0.3" + meow "^7.0.1" + resolve-from "^5.0.0" + gensync@^1.0.0-beta.1, gensync@^1.0.0-beta.2: version "1.0.0-beta.2" resolved "https://registry.yarnpkg.com/gensync/-/gensync-1.0.0-beta.2.tgz#32a6ee76c3d7f52d46b2b1ae5d93fea8580a25e0" @@ -14569,6 +14581,23 @@ memorystream@^0.3.1: resolved "https://registry.yarnpkg.com/memorystream/-/memorystream-0.3.1.tgz#86d7090b30ce455d63fbae12dda51a47ddcaf9b2" integrity sha1-htcJCzDORV1j+64S3aUaR93K+bI= +meow@^7.0.1: + version "7.1.1" + resolved "https://registry.yarnpkg.com/meow/-/meow-7.1.1.tgz#7c01595e3d337fcb0ec4e8eed1666ea95903d306" + integrity sha512-GWHvA5QOcS412WCo8vwKDlTelGLsCGBVevQB5Kva961rmNfun0PCbv5+xta2kUMFJyR8/oWnn7ddeKdosbAPbA== + dependencies: + "@types/minimist" "^1.2.0" + camelcase-keys "^6.2.2" + decamelize-keys "^1.1.0" + hard-rejection "^2.1.0" + minimist-options "4.1.0" + normalize-package-data "^2.5.0" + read-pkg-up "^7.0.1" + redent "^3.0.0" + trim-newlines "^3.0.0" + type-fest "^0.13.1" + yargs-parser "^18.1.3" + meow@^8.0.0: version "8.1.2" resolved "https://registry.yarnpkg.com/meow/-/meow-8.1.2.tgz#bcbe45bda0ee1729d350c03cffc8395a36c4e897" @@ -19338,6 +19367,14 @@ ripemd160@^2.0.0, ripemd160@^2.0.1: hash-base "^3.0.0" inherits "^2.0.1" +robotstxt-webpack-plugin@^7.0.0: + version "7.0.0" + resolved "https://registry.yarnpkg.com/robotstxt-webpack-plugin/-/robotstxt-webpack-plugin-7.0.0.tgz#04b26c9e1cd2ad8ebc5e20918aeeb6ed5d56835e" + integrity sha512-Rk0moeHvqc3to6oE2z5lKW9GTJyQC/ysogNy465N2CT2Xvp2/DMPkix8idnpmBLfExrQjml18B8Z/HoAC/yPlg== + dependencies: + generate-robotstxt "^8.0.0" + webpack-sources "^1.4.3" + router-ips@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/router-ips/-/router-ips-1.0.0.tgz#44e00858ebebc0133d58e40b2cd8a1fbb04203f5" @@ -21409,6 +21446,11 @@ type-fest@^0.12.0: resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.12.0.tgz#f57a27ab81c68d136a51fd71467eff94157fa1ee" integrity sha512-53RyidyjvkGpnWPMF9bQgFtWp+Sl8O2Rp13VavmJgfAP9WWG6q6TkrKU8iyJdnwnfgHI6k2hTlgqH4aSdjoTbg== +type-fest@^0.13.1: + version "0.13.1" + resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.13.1.tgz#0172cb5bce80b0bd542ea348db50c7e21834d934" + integrity sha512-34R7HTnG0XIJcBSn5XhDd7nNFPRcXYRZrBB2O2jdKqYODldSzBAqzsWoZYYvduky73toYS/ESqxPvkDf/F0XMg== + type-fest@^0.16.0: version "0.16.0" resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.16.0.tgz#3240b891a78b0deae910dbeb86553e552a148860" @@ -22782,7 +22824,7 @@ yargs-parser@^13.1.2: camelcase "^5.0.0" decamelize "^1.2.0" -yargs-parser@^18.1.2: +yargs-parser@^18.1.2, yargs-parser@^18.1.3: version "18.1.3" resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-18.1.3.tgz#be68c4975c6b2abf469236b0c870362fab09a7b0" integrity sha512-o50j0JeToy/4K6OZcaQmW6lyXXKhq7csREXcDwk2omFPJEwUNOVtJKvmDr9EI1fAJZUyZcRF7kxGBWmRXudrCQ== From 5d987af163e9b65397d5d9a50bbc1fb1aa8ace74 Mon Sep 17 00:00:00 2001 From: Zakaria Mansouri Date: Sun, 7 Aug 2022 20:05:37 +0200 Subject: [PATCH 2/3] moved env up to job level --- .github/workflows/cd.deploy.stg.yml | 16 ++++++++-------- .github/workflows/cd.deploy.yml | 16 ++++++++-------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/cd.deploy.stg.yml b/.github/workflows/cd.deploy.stg.yml index 4beef5680..915ed7e33 100644 --- a/.github/workflows/cd.deploy.stg.yml +++ b/.github/workflows/cd.deploy.stg.yml @@ -14,6 +14,14 @@ jobs: node-version: [14.x] os: [ubuntu-latest] + env: + CI: true + STAGE: staging + SSH_KNOWN_HOSTS: ${{ secrets.SSH_KNOWN_HOSTS }} + SSH_ADDRESS_STG: ${{ secrets.SSH_ADDRESS_STG }} + FIREBASE_TOKEN: ${{ secrets.FIREBASE_TOKEN }} + EXPO_TOKEN: ${{ secrets.EXPO_TOKEN }} + steps: - name: "Git" uses: actions/checkout@v2 @@ -60,11 +68,3 @@ jobs: shell: bash run: | yarn deploy:stg - - env: - CI: true - STAGE: "staging" - SSH_KNOWN_HOSTS: ${{ secrets.SSH_KNOWN_HOSTS }} - SSH_ADDRESS_STG: ${{ secrets.SSH_ADDRESS_STG }} - FIREBASE_TOKEN: ${{ secrets.FIREBASE_TOKEN }} - EXPO_TOKEN: ${{ secrets.EXPO_TOKEN }} diff --git a/.github/workflows/cd.deploy.yml b/.github/workflows/cd.deploy.yml index ab0ea2a2f..9ceff3f91 100644 --- a/.github/workflows/cd.deploy.yml +++ b/.github/workflows/cd.deploy.yml @@ -14,6 +14,14 @@ jobs: node-version: [14.x] os: [ubuntu-latest] + env: + CI: true + STAGE: production + SSH_KNOWN_HOSTS: ${{ secrets.SSH_KNOWN_HOSTS }} + SSH_ADDRESS_PRD: ${{ secrets.SSH_ADDRESS_PRD }} + FIREBASE_TOKEN: ${{ secrets.FIREBASE_TOKEN }} + EXPO_TOKEN: ${{ secrets.EXPO_TOKEN }} + steps: - name: "Git" uses: actions/checkout@v2 @@ -67,11 +75,3 @@ jobs: shell: bash run: | yarn deploy - - env: - CI: true - STAGE: "production" - SSH_KNOWN_HOSTS: ${{ secrets.SSH_KNOWN_HOSTS }} - SSH_ADDRESS_PRD: ${{ secrets.SSH_ADDRESS_PRD }} - FIREBASE_TOKEN: ${{ secrets.FIREBASE_TOKEN }} - EXPO_TOKEN: ${{ secrets.EXPO_TOKEN }} From d13528a10aa2c842b899f1c4b6c9ad5b0598c949 Mon Sep 17 00:00:00 2001 From: Zakaria Mansouri Date: Sun, 7 Aug 2022 20:10:54 +0200 Subject: [PATCH 3/3] Added robot.txt to disallow indexing of api server --- api/package.json | 1 + api/src/app/index.ts | 3 +++ yarn.lock | 5 +++++ 3 files changed, 9 insertions(+) diff --git a/api/package.json b/api/package.json index 2ab5d7940..88146080f 100644 --- a/api/package.json +++ b/api/package.json @@ -19,6 +19,7 @@ "dotenv": "^8.2.0", "express": "^4.17.1", "express-rate-limit": "^5.2.6", + "express-robots-txt": "^1.0.0", "fs-extra": "^10.0.0", "helmet": "^4.4.1", "make-fetch-happen": "^9.0.2", diff --git a/api/src/app/index.ts b/api/src/app/index.ts index 41a8921bd..829699f3f 100644 --- a/api/src/app/index.ts +++ b/api/src/app/index.ts @@ -19,6 +19,7 @@ import { LoggerMiddleware } from "./middlewares/logger"; import { SecurityMiddleware } from "./middlewares/security"; import { SentryErrorHandlerMiddleware } from "./middlewares/sentry-error-handler"; import { SentryRequestHandlerMiddleware } from "./middlewares/sentry-request-handler"; +const robots = require("express-robots-txt"); // eslint-disable-line @typescript-eslint/no-var-requires // Use typedi container useContainer(Container); @@ -59,6 +60,8 @@ const app: Application = createExpressServer(routingControllersOptions); const logger = Container.get(LoggerService); +app.use(robots({ UserAgent: "*", Disallow: "/" })); + // Start it app.listen(PORT, () => { const commonConfig = fsConfig(NODE_ENV); diff --git a/yarn.lock b/yarn.lock index 98f9b8dcc..4df4c6348 100644 --- a/yarn.lock +++ b/yarn.lock @@ -9711,6 +9711,11 @@ express-rate-limit@^5.2.6: resolved "https://registry.yarnpkg.com/express-rate-limit/-/express-rate-limit-5.5.1.tgz#110c23f6a65dfa96ab468eda95e71697bc6987a2" integrity sha512-MTjE2eIbHv5DyfuFz4zLYWxpqVhEhkTiwFGuB74Q9CSou2WHO52nlE5y3Zlg6SIsiYUIPj6ifFxnkPz6O3sIUg== +express-robots-txt@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/express-robots-txt/-/express-robots-txt-1.0.0.tgz#354dde6f241c3d4ed39f74cfd0604dd92daea497" + integrity sha512-ZzXTJgvHY6N8MIPzWidIrOGngkNJ8nrSP9Ox0nztVkQxvpwcX8egJvhM6tBUojf9xDmnSc7J5RlWQJJK91N3aA== + express-session@^1.17.1: version "1.17.2" resolved "https://registry.yarnpkg.com/express-session/-/express-session-1.17.2.tgz#397020374f9bf7997f891b85ea338767b30d0efd"