updates proxy to support access control allow list (#3407)

* feat: updates proxy to support access control allow list

* fix: remove downstream access-control-allow-origin

* fix: update readme for m1

* fix: move purge call to the backend

* fix: test fix and add await

* fix: moving cache purge to helper

---------

Co-authored-by: Morgan Ludtke <ludtkemorgan@gmail.com>
Co-authored-by: Yazeed Loonat <yazeedloonat@gmail.com>

backend/core/.env.template

@@ -9,6 +9,7 @@ APP_SECRET='SOME-LONG-SECRET-KEY'
 CLOUDINARY_SECRET=CLOUDINARY_SECRET
 CLOUDINARY_KEY=CLOUDINARY_KEY
 PARTNERS_BASE_URL=http://localhost:3001
+PROXY_URL=
 NEW_RELIC_APP_NAME=Bloom Backend Local
 NEW_RELIC_LICENSE_KEY=
 GOOGLE_API_ID=

backend/core/package.json

@@ -40,6 +40,7 @@
   "dependencies": {
     "@anchan828/nest-sendgrid": "^0.3.25",
     "@google-cloud/translate": "^6.2.6",
+    "@nestjs/axios": "2.0.0",
     "@nestjs/cli": "^8.2.1",
     "@nestjs/common": "^8.3.1",
     "@nestjs/config": "^1.2.0",

backend/core/src/listings/listings.module.ts

@@ -1,4 +1,5 @@
 import { Logger, Module } from "@nestjs/common"
+import { HttpModule } from "@nestjs/axios"
 import { TypeOrmModule } from "@nestjs/typeorm"
 import { ListingsService } from "./listings.service"
 import { ListingsController } from "./listings.controller"
@@ -32,6 +33,7 @@ import { ListingsCronService } from "./listings-cron.service"
     TranslationsModule,
     ActivityLogModule,
     ApplicationFlaggedSetsModule,
+    HttpModule,
   ],
   providers: [ListingsService, ListingsCronService, Logger],
   exports: [ListingsService],

backend/core/src/listings/listings.service.ts

@@ -1,4 +1,5 @@
 import { Inject, Injectable, NotFoundException, Scope } from "@nestjs/common"
+import { HttpService } from "@nestjs/axios"
 import { InjectRepository } from "@nestjs/typeorm"
 import { Pagination } from "nestjs-typeorm-paginate"
 import { In, Repository } from "typeorm"
@@ -20,6 +21,7 @@ import { REQUEST } from "@nestjs/core"
 import { User } from "../auth/entities/user.entity"
 import { ApplicationFlaggedSetsService } from "../application-flagged-sets/application-flagged-sets.service"
 import { ListingsQueryBuilder } from "./db/listing-query-builder"
+import { firstValueFrom } from "rxjs"
 
 @Injectable({ scope: Scope.REQUEST })
 export class ListingsService {
@@ -29,7 +31,8 @@ export class ListingsService {
     private readonly translationService: TranslationsService,
     private readonly authzService: AuthzService,
     @Inject(REQUEST) private req: ExpressRequest,
-    private readonly afsService: ApplicationFlaggedSetsService
+    private readonly afsService: ApplicationFlaggedSetsService,
+    private readonly httpService: HttpService
   ) {}
 
   private getFullyJoinedQueryBuilder() {
@@ -143,7 +146,9 @@ export class ListingsService {
           : listing.closedAt,
     })
 
-    return await this.listingRepository.save(listing)
+    const saveResponse = await this.listingRepository.save(listing)
+    await this.cachePurge(listing, listingDto, saveResponse)
+    return saveResponse
   }
 
   async delete(listingId: string) {
@@ -218,4 +223,39 @@ export class ListingsService {
 
     return result
   }
+
+  /**
+   * Send purge request to Nginx.
+   * Wrapped in try catch, because it's possible that content may not be cached in between edits,
+   * and will return a 404, which is expected.
+   * listings* purges all /listings locations (with args, details), so if we decide to clear on certain locations,
+   * like all lists and only the edited listing, then we can do that here (with a corresponding update to nginx config)
+   */
+  private async cachePurge(
+    currentListing: Listing,
+    incomingChanges: ListingCreateDto | ListingUpdateDto,
+    saveReponse: Listing
+  ) {
+    if (process.env.PROXY_URL) {
+      await firstValueFrom(
+        this.httpService.request({
+          baseURL: process.env.PROXY_URL,
+          method: "PURGE",
+          url: `/listings/${saveReponse.id}*`,
+        })
+      ).catch((e) => console.log(`purge listing ${saveReponse.id} error = `, e))
+      if (
+        incomingChanges.status !== ListingStatus.pending ||
+        currentListing.status === ListingStatus.active
+      ) {
+        await firstValueFrom(
+          this.httpService.request({
+            baseURL: process.env.PROXY_URL,
+            method: "PURGE",
+            url: "/listings?*",
+          })
+        ).catch((e) => console.log("purge all listings error = ", e))
+      }
+    }
+  }
 }

backend/core/src/listings/tests/listings.service.spec.ts

@@ -14,6 +14,7 @@ import { ApplicationFlaggedSetsService } from "../../application-flagged-sets/ap
 import { ListingRepository } from "../db/listing.repository"
 import { ListingsQueryBuilder } from "../db/listing-query-builder"
 import { UserRepository } from "../../auth/repositories/user-repository"
+import { HttpService } from "@nestjs/axios"
 
 /* eslint-disable @typescript-eslint/unbound-method */
 
@@ -153,6 +154,10 @@ describe("ListingsService", () => {
           provide: getRepositoryToken(AmiChart),
           useValue: jest.fn(),
         },
+        {
+          provide: HttpService,
+          useValue: jest.fn(),
+        },
         {
           provide: TranslationsService,
           useValue: { translateListing: jest.fn() },

backend/core/src/shared/shared.module.ts

@@ -12,6 +12,7 @@ import Joi from "joi"
           .default("development"),
         EMAIL_API_KEY: Joi.string().required(),
         DATABASE_URL: Joi.string().required(),
+        PROXY_URL: Joi.string(),
         THROTTLE_TTL: Joi.number().default(1),
         THROTTLE_LIMIT: Joi.number().default(100),
         APP_SECRET: Joi.string().required().min(16),

backend/proxy/Dockerfile

@@ -1,4 +1,4 @@
-FROM nginx:1.11
+FROM nginx:1.19.0
 
 MAINTAINER David Galoyan <davojan@gmail.com>
 
@@ -7,13 +7,13 @@ ENV NGX_CACHE_PURGE_VERSION=2.4.1
 # Install basic packages and build tools
 RUN apt-get update && \
     apt-get install --no-install-recommends --no-install-suggests -y \
-      wget \
-      build-essential \
-      libssl-dev \
-      libpcre3 \
-      zlib1g \
-      zlib1g-dev \
-      libpcre3-dev && \
+    wget \
+    build-essential \
+    libssl-dev \
+    libpcre3 \
+    zlib1g \
+    zlib1g-dev \
+    libpcre3-dev && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
@@ -22,7 +22,7 @@ RUN NGINX_VERSION=`nginx -V 2>&1 | grep "nginx version" | awk -F/ '{ print $2}'`
     cd /tmp && \
     wget http://nginx.org/download/nginx-$NGINX_VERSION.tar.gz && \
     wget https://github.com/nginx-modules/ngx_cache_purge/archive/$NGX_CACHE_PURGE_VERSION.tar.gz \
-         -O ngx_cache_purge-$NGX_CACHE_PURGE_VERSION.tar.gz && \
+    -O ngx_cache_purge-$NGX_CACHE_PURGE_VERSION.tar.gz && \
     tar -xf nginx-$NGINX_VERSION.tar.gz && \
     mv nginx-$NGINX_VERSION nginx && \
     rm nginx-$NGINX_VERSION.tar.gz && \
@@ -42,6 +42,6 @@ RUN cd /tmp/nginx && \
     make && make install && \
     rm -rf /tmp/nginx* \
 
-ENV PROTOCOL=https
+    ENV PROTOCOL=https
 
-CMD /bin/bash -c "envsubst '\$PORT,\$BACKEND_HOSTNAME,\$PROTOCOL' < /etc/nginx/conf.d/default.conf.template > /etc/nginx/conf.d/default.conf && envsubst '\$PORT,\$BACKEND_HOSTNAME,\$PROTOCOL' < /etc/nginx/conf.d/shared-location.conf.template > /etc/nginx/shared-location.conf" && nginx -g 'daemon off;'
+CMD /bin/bash -c "envsubst '\$PORT,\$BACKEND_HOSTNAME,\$PROTOCOL,\$ALLOW_LIST' < /etc/nginx/conf.d/default.conf.template > /etc/nginx/conf.d/default.conf && envsubst '\$PORT,\$BACKEND_HOSTNAME,\$PROTOCOL,\$ALLOW_LIST' < /etc/nginx/conf.d/shared-location.conf.template > /etc/nginx/shared-location.conf" && nginx -g 'daemon off;'

backend/proxy/README.md

@@ -3,6 +3,27 @@
 We want to serve different versions of the API under different paths e.g. `/v2`, `/v3` but not necessarily reflect that convention in the code.
 To achieve that an NGINX proxy has been created and set up as an entrypoint to the entire API. It provides path level routing e.g. `/v2` will be routed to a different heroku APP then `/`.
 
+### Running Locally with Docker Desktop
+
+# make sure Docker Desktop is started
+
+# build the image: `docker build --pull --rm -f "backend/proxy/Dockerfile" -t bloom:latest "backend/proxy"`
+
+# In the Docker Desktop Dashboard, go to "Images", find the image and click "Run" (play button)
+
+# Configure Optional Settings before running:
+
+# Ports -> Host Port: 9000
+
+# Environment Variables:
+
+    - PORT: 80
+    - BACKEND_HOSTNAME: [Your local IP]:3100 (e.g. 192.168.86.231:3100)
+    - PROTOCOL: http
+    - ALLOW_LIST: localhost:3000|localhost:3001
+
+# Run and you should be able to access listings at http://localhost:9000/listings
+
 ### Setup
 
 Based on [this tutorial](https://dashboard.heroku.com/apps/bloom-reference-backend-proxy/deploy/heroku-container). All values are for `bloom-reference-backend-proxy` and each environment requires it's own proxy.
@@ -31,6 +52,8 @@ Now you can sign into Container Registry.
 $ heroku container:login
 ```
 
+_Note_ if you are using an Apple M1 device the following steps will not work. You will have to do the following: https://stackoverflow.com/a/67001433
+
 Push your Docker-based app
 Build the Dockerfile in the current directory and push the Docker image.
 
@@ -48,9 +71,9 @@ $ heroku container:release --app bloom-reference-backend-proxy web
 
 #### Configuration
 
-Heroku Proxy app requires two environment variables to work:
+Heroku Proxy app requires four environment variables to work:
 
-```
-heroku config:set --app bloom-reference-backend-proxy BACKEND_V1_HOSTNAME=example.v1.hostname.com
-heroku config:set --app bloom-reference-backend-proxy BACKEND_V2_HOSTNAME=example.v2.hostname.com
-```
+PORT: 443
+BACKEND_HOSTNAME: bloom-backend-orm.herokuapp.com
+PROTOCOL: https
+ALLOW_LIST: partners.bloom.exygy.dev|bloom.exygy.dev

backend/proxy/default.conf

@@ -1,6 +1,6 @@
 proxy_cache_path /tmp/cache_nginx/ levels=1:2 keys_zone=webapp_cache:10m max_size=10g inactive=1440 use_temp_path=off;
 
-log_format upstreamlog '[$time_local] $remote_addr - $remote_user - $server_name $host to: $upstream_addr: $request $status upstream_response_time $upstream_response_time msec $msec request_time $request_time';
+log_format upstreamlog '[$time_local] $http_origin $remote_addr - $remote_user - $server_name $host to: $upstream_addr: $request $status upstream_response_time $upstream_response_time msec $msec request_time $request_time';
 
 server {
   listen $PORT;

backend/proxy/shared-location.conf

@@ -1,16 +1,19 @@
-    proxy_cache_min_uses 1;
-    proxy_cache_revalidate on;
-    proxy_cache_background_update on;
-    proxy_cache_lock on;
-    proxy_ssl_server_name on;
-    proxy_cache webapp_cache;
-    proxy_cache_use_stale error timeout http_500 http_502 http_503 http_504;
-    proxy_cache_key $uri$is_args$args$http_language;
-    if ($request_method = 'PURGE') {
-      # TODO: make vairable that's passed in for allow origin purge
-      add_header Access-Control-Allow-Origin *;
-    }
-    add_header X-Cache-Status $upstream_cache_status;
-    add_header Access-Control-Allow-Headers 'Content-Type, X-Language, X-JurisdictionName, Authorization';
-    add_header Access-Control-Allow-Methods 'GET, POST, OPTIONS, PUT, DELETE, PURGE';
-    proxy_pass $PROTOCOL://$BACKEND_HOSTNAME;
+proxy_cache_min_uses 1;
+proxy_cache_revalidate on;
+proxy_cache_background_update on;
+proxy_cache_lock on;
+proxy_ssl_server_name on;
+proxy_cache webapp_cache;
+proxy_cache_use_stale error timeout http_500 http_502 http_503 http_504;
+proxy_cache_key $uri$is_args$args$http_language;
+proxy_hide_header Access-Control-Allow-Origin;
+if ($http_origin ~* "^https?://($ALLOW_LIST)$") {
+    add_header Access-Control-Allow-Origin *;
+}
+if ($request_method = 'PURGE') {
+    add_header Access-Control-Allow-Origin *;
+}
+add_header X-Cache-Status $upstream_cache_status;
+add_header Access-Control-Allow-Headers 'Content-Type, X-Language, X-JurisdictionName, Authorization';
+add_header Access-Control-Allow-Methods 'GET, POST, OPTIONS, PUT, DELETE, PURGE';
+proxy_pass $PROTOCOL://$BACKEND_HOSTNAME;

sites/partners/src/components/listings/PaperListingForm/index.tsx

@@ -1,5 +1,4 @@
 import React, { useState, useCallback, useContext, useEffect } from "react"
-import axios from "axios"
 import { useRouter } from "next/router"
 import dayjs from "dayjs"
 import {
@@ -199,32 +198,6 @@ const ListingForm = ({ listing, editMode }: ListingFormProps) => {
           reset(formData)
 
           if (result) {
-            /**
-             * Send purge request to Nginx.
-             * Wrapped in try catch, because it's possible that content may not be cached in between edits,
-             * and will return a 404, which is expected.
-             * listings* purges all /listings locations (with args, details), so if we decide to clear on certain locations,
-             * like all lists and only the edited listing, then we can do that here (with a corresponding update to nginx config)
-             */
-            if (process.env.backendProxyBase) {
-              try {
-                // clear individual listing's cache
-                await axios.request({
-                  url: `${process.env.backendProxyBase}/listings/${result.id}*`,
-                  method: "purge",
-                })
-                // clear list caches if published
-                if (result.status !== ListingStatus.pending) {
-                  await axios.request({
-                    url: `${process.env.backendProxyBase}/listings?*`,
-                    method: "purge",
-                  })
-                }
-              } catch (e) {
-                console.log("purge error = ", e)
-              }
-            }
-
             setSiteAlertMessage(
               editMode ? t("listings.listingUpdated") : t("listings.listingSubmitted"),
               "success"

yarn.lock

@@ -2984,6 +2984,11 @@
     strict-event-emitter "^0.2.4"
     web-encoding "^1.1.5"
 
+"@nestjs/axios@2.0.0":
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/@nestjs/axios/-/axios-2.0.0.tgz#2116fad483e232ef102a877b503a9f19926bd102"
+  integrity sha512-F6oceoQLEn031uun8NiommeMkRIojQqVryxQy/mK7fx0CI0KbgkJL3SloCQcsOD+agoEnqKJKXZpEvL6FNswJg==
+
 "@nestjs/cli@^8.2.1":
   version "8.2.1"
   resolved "https://registry.npmjs.org/@nestjs/cli/-/cli-8.2.1.tgz"