Merge branch 'AI-250-debug-open-files' into 'master'

Add logging for AI-250 "Debug open files"

See merge request developers/ask-izzy!640

.gitlab-ci.yml

@@ -13,7 +13,7 @@ default:
   before_script:
     - VERSION=$(git describe | sed 's/-g/-/') && echo $VERSION
     - TAG_DEVELOPMENT_STAGE=${DOCKER_REPO}:$VERSION-dev && echo $TAG_DEVELOPMENT_STAGE
-    - TAG_DISTRIBUTION_STAGE=${DOCKER_REPO}:$VERSION-dist && echo $TAG_DISTRIBUTION_STAGE
+    - TAG_DISTRIBUTION_STAGE=${DOCKER_REPO}:${CI_COMMIT_TAG:="$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA"} && echo $TAG_DISTRIBUTION_STAGE
     - TAG_PRODUCTION=${DOCKER_REPO}:$VERSION && echo $TAG_PRODUCTION
     - TAG_CACHE_DEV_STAGE=${DOCKER_REPO}:cache-$CI_COMMIT_BRANCH-dev && echo $TAG_CACHE_DEV_STAGE
     - TAG_CACHE_DIST_STAGE=${DOCKER_REPO}:cache-$CI_COMMIT_BRANCH-dist && echo $TAG_CACHE_DIST_STAGE

Dockerfile

@@ -60,7 +60,9 @@ RUN mkdir /app && \
     apt-get -y update && \
     apt-get -y --no-install-recommends install \
         # Used by start-server-and-test
-        procps && \
+        procps \
+        # Used for open fd debugging
+        lsof && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 

docker-compose.emulate-production.yml

@@ -8,13 +8,11 @@ services:
         - VERSION=emulated-production
     command: "shell bash -c 'yarn run deploy && yarn run serve'"
     environment:
-      SITE_PROTOCOL: "http"
-      SITE_DOMAIN: "localhost"
+      SITE_PROTOCOL: "https"
+      SITE_DOMAIN: "askizzy.org.au"
       STATIC_PAGE_REGENERATION_SECRET: "hunter2"
       STATIC_PAGE_PREVIEW_SECRET: "hunter2"
       ENVIRONMENT: "emulated production"
-      SITE_PROTOCOL: "https"
-      SITE_DOMAIN: "askizzy.org.au"
       HTTP_PROXY: "http://http-proxy:8888"
     volumes:
       - ./.env.production.local:/app/.env.production.local
@@ -34,5 +32,12 @@ services:
     command: sh -c "sed -i -e 's/^LogLevel.*/LogLevel Warning/' /etc/tinyproxy/tinyproxy.conf && tinyproxy -d"
     restart: unless-stopped
 
+  set-volume-permissions:
+    build:
+      context: .
+      target: set-volume-permissions
+    volumes:
+      - storage:/storage
+
 volumes:
   storage:

package.json

@@ -115,6 +115,7 @@
     "js-yaml": "3.14.1",
     "jsonlint": "1.6.3",
     "just-debounce-it": "^2.0.0",
+    "list-open-files": "^1.1.0",
     "mocha": "^6.0.0",
     "moment": "2.24.0",
     "moment-timezone": "0.5.25",

pages/api/external-resource-proxy/[[...path]].js

@@ -3,6 +3,7 @@
 // https://github.com/vercel/next.js/issues/37028 stopping us
 // proxying the google maps api
 import { createProxyMiddleware } from "http-proxy-middleware";
+import { lsof } from "list-open-files"
 
 import { unflattenDomain } from "@/helpers/url.helpers"
 
@@ -17,6 +18,23 @@ export default function handler(req: any, res: any): void {
     if (!process.env.DOMAINS_TO_PROXY) {
         return res.status(400).send("Route not available");
     }
+
+    setTimeout(async() => {
+        const processInfos = await lsof()
+
+        const files = processInfos[0]?.files
+        if (!files) {
+            return
+        }
+        const sockets = files.filter(file => (file.type === "IP") && file.from)
+        const incomingSockets = sockets.filter(socket => socket.from.port === 8000)
+        const outgoingSockets = sockets.filter(socket => socket.from.port !== 8000)
+
+        console.log(
+            `${files.length} open fd, ${incomingSockets.length} incoming sockets, ${outgoingSockets.length} outgoing`
+        )
+    }, 1000)
+
     const targetBaseUrl = getTargetBaseUrl(req)
 
     if (!allowedDomains.has(targetBaseUrl.host)) {

pages/api/open-fd-debug.js

@@ -0,0 +1,30 @@
+/* @flow */
+import { lsof } from "list-open-files"
+export default async function handler(req: any, res: any): Promise<void> {
+    // Obviously not super secure but we're only trying to make this
+    // slightly more secuire than an unlisted url
+    const { auth } = req.query
+    if (!process.env.STATIC_PAGE_REGENERATION_SECRET || (auth !== process.env.STATIC_PAGE_REGENERATION_SECRET)) {
+        return res.status(404).send("404")
+    }
+
+    try {
+        const processInfos = await lsof()
+
+        const files = processInfos[0]?.files
+        if (!files) {
+            return res.status(500).send("no files")
+        }
+        const sockets = files.filter(file => (file.type === "IP") && file.from)
+        const nonSockets = files.filter(file => (file.type !== "IP") || !file.from)
+        const incomingSockets = sockets.filter(socket => socket.from.port === 8000)
+        const outgoingSockets = sockets.filter(socket => socket.from.port !== 8000)
+        return res.status(200).send({
+            incomingSockets,
+            outgoingSockets,
+            nonSockets,
+        })
+    } catch (error) {
+        return res.status(500).send("error: " + error)
+    }
+}

yarn.lock

@@ -9919,6 +9919,11 @@ lines-and-columns@^1.1.6:
   resolved "https://registry.yarnpkg.com/lines-and-columns/-/lines-and-columns-1.2.4.tgz#eca284f75d2965079309dc0ad9255abb2ebc1632"
   integrity sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==
 
+list-open-files@^1.1.0:
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/list-open-files/-/list-open-files-1.1.0.tgz#df824dbdb145444cc774b0c3e7200c65c5392da6"
+  integrity sha512-HdpgU3D+hBHWclA5J6Uyi3S8VhF+k3+qXur8gOmEuiBltCtUyr0z+HnJb63kNf1kJHFiZ00TWfRF/4m95wfblQ==
+
 lit-element@^3.3.0:
   version "3.3.0"
   resolved "https://registry.yarnpkg.com/lit-element/-/lit-element-3.3.0.tgz#a8e0b9ca5239faf721d9af4460f106cf7f03533b"