[pull] main from overleaf:main

server-ce/Dockerfile

@@ -37,6 +37,9 @@ RUN node genScript compile | bash
 # --------------------------------------------------
 ADD server-ce/runit /etc/service
 
+# Copy runit global settings to its location
+# ------------------------------------------
+ADD server-ce/config/env.sh /etc/overleaf/env.sh
 
 # Configure nginx
 # ---------------

server-ce/config/env.sh

@@ -0,0 +1,14 @@
+export CHAT_HOST=127.0.0.1
+export CLSI_HOST=127.0.0.1
+export CONTACTS_HOST=127.0.0.1
+export DOCSTORE_HOST=127.0.0.1
+export DOCUMENT_UPDATER_HOST=127.0.0.1
+export DOCUPDATER_HOST=127.0.0.1
+export FILESTORE_HOST=127.0.0.1
+export HISTORY_V1_HOST=127.0.0.1
+export NOTIFICATIONS_HOST=127.0.0.1
+export PROJECT_HISTORY_HOST=127.0.0.1
+export REALTIME_HOST=127.0.0.1
+export SPELLING_HOST=127.0.0.1
+export WEB_HOST=127.0.0.1
+export WEB_API_HOST=127.0.0.1

server-ce/config/settings.js

@@ -256,16 +256,16 @@ const settings = {
 
   apis: {
     web: {
-      url: 'http://localhost:3000',
+      url: 'http://127.0.0.1:3000',
       user: httpAuthUser,
       pass: httpAuthPass,
     },
     project_history: {
       sendProjectStructureOps: true,
-      url: 'http://localhost:3054',
+      url: 'http://127.0.0.1:3054',
     },
     v1_history: {
-      url: process.env.V1_HISTORY_URL || 'http://localhost:3100/api',
+      url: process.env.V1_HISTORY_URL || 'http://127.0.0.1:3100/api',
       user: 'staging',
       pass: process.env.STAGING_PASSWORD,
       requestTimeout: parseInt(
@@ -409,7 +409,7 @@ if (
 
 if (parse(process.env.OVERLEAF_IS_SERVER_PRO) === true) {
   settings.bypassPercentageRollouts = true
-  settings.apis.references = { url: 'http://localhost:3040' }
+  settings.apis.references = { url: 'http://127.0.0.1:3040' }
 }
 
 // Compiler

server-ce/cron/deactivate-projects.sh

@@ -14,7 +14,7 @@ if [[ "${ENABLE_CRON_RESOURCE_DELETION:-null}" != "true" ]]; then
   exit 0
 fi
 
-WEB_URL='http://localhost:3000'
+WEB_URL='http://127.0.0.1:3000'
 
 USER=$(cat /etc/container_environment/WEB_API_USER)
 PASS=$(cat /etc/container_environment/WEB_API_PASSWORD)

server-ce/cron/delete-projects.sh

@@ -14,7 +14,7 @@ if [[ "${ENABLE_CRON_RESOURCE_DELETION:-null}" != "true" ]]; then
   exit 0
 fi
 
-WEB_URL='http://localhost:3000'
+WEB_URL='http://127.0.0.1:3000'
 
 USER=$(cat /etc/container_environment/WEB_API_USER)
 PASS=$(cat /etc/container_environment/WEB_API_PASSWORD)

server-ce/cron/delete-users.sh

@@ -14,7 +14,7 @@ if [[ "${ENABLE_CRON_RESOURCE_DELETION:-null}" != "true" ]]; then
   exit 0
 fi
 
-WEB_URL='http://localhost:3000'
+WEB_URL='http://127.0.0.1:3000'
 
 USER=$(cat /etc/container_environment/WEB_API_USER)
 PASS=$(cat /etc/container_environment/WEB_API_PASSWORD)

server-ce/cron/project-history-periodic-flush.sh

@@ -7,6 +7,6 @@ echo "Flush project-history queue"
 echo "--------------------------"
 date
 
-PROJECT_HISTORY_URL='http://localhost:3054'
+PROJECT_HISTORY_URL='http://127.0.0.1:3054'
 
 curl -X POST "${PROJECT_HISTORY_URL}/flush/old?timeout=3600000&limit=5000&background=1"

server-ce/cron/project-history-retry-hard.sh

@@ -7,6 +7,6 @@ echo "Retry project-history errors (hard)"
 echo "-----------------------------------"
 date
 
-PROJECT_HISTORY_URL='http://localhost:3054'
+PROJECT_HISTORY_URL='http://127.0.0.1:3054'
 
 curl -X POST "${PROJECT_HISTORY_URL}/retry/failures?failureType=hard&timeout=3600000&limit=10000"

server-ce/cron/project-history-retry-soft.sh

@@ -6,6 +6,6 @@ echo "-----------------------------------"
 echo "Retry project-history errors (soft)"
 echo "-----------------------------------"
 
-PROJECT_HISTORY_URL='http://localhost:3054'
+PROJECT_HISTORY_URL='http://127.0.0.1:3054'
 
 curl -X POST "${PROJECT_HISTORY_URL}/retry/failures?failureType=soft&timeout=3600000&limit=10000"

server-ce/nginx/overleaf.conf

@@ -10,7 +10,7 @@ server {
 	}
 
 	location / {
-		proxy_pass http://localhost:4000;
+		proxy_pass http://127.0.0.1:4000;
 		proxy_http_version 1.1;
 		proxy_set_header Upgrade $http_upgrade;
 		proxy_set_header Connection "upgrade";
@@ -22,7 +22,7 @@ server {
 	}
 
 	location /socket.io {
-		proxy_pass http://localhost:3026;
+		proxy_pass http://127.0.0.1:3026;
 		proxy_http_version 1.1;
 		proxy_set_header Upgrade $http_upgrade;
 		proxy_set_header Connection "upgrade";
@@ -48,22 +48,22 @@ server {
 
   # handle output files for specific users
   location ~ ^/project/([0-9a-f]+)/user/([0-9a-f]+)/build/([0-9a-f-]+)/output/output\.([a-z]+)$ {
-		proxy_pass http://localhost:8080; # clsi-nginx.conf
+		proxy_pass http://127.0.0.1:8080; # clsi-nginx.conf
 		proxy_http_version 1.1;
   }
   # handle output files for anonymous users
   location ~ ^/project/([0-9a-f]+)/build/([0-9a-f-]+)/output/output\.([a-z]+)$ {
-		proxy_pass http://localhost:8080; # clsi-nginx.conf
+		proxy_pass http://127.0.0.1:8080; # clsi-nginx.conf
 		proxy_http_version 1.1;
   }
   # PDF range for specific users
   location ~ ^/project/([0-9a-f]+)/user/([0-9a-f]+)/content/([0-9a-f-]+/[0-9a-f]+)$ {
-		proxy_pass http://localhost:8080; # clsi-nginx.conf
+		proxy_pass http://127.0.0.1:8080; # clsi-nginx.conf
 		proxy_http_version 1.1;
   }
   # PDF range for anonymous users
   location ~ ^/project/([0-9a-f]+)/content/([0-9a-f-]+/[0-9a-f]+)$ {
-		proxy_pass http://localhost:8080; # clsi-nginx.conf
+		proxy_pass http://127.0.0.1:8080; # clsi-nginx.conf
 		proxy_http_version 1.1;
   }
 

server-ce/runit/chat-overleaf/run

@@ -6,4 +6,7 @@ if [ "$DEBUG_NODE" == "true" ]; then
     NODE_PARAMS="--inspect=0.0.0.0:30100"
 fi
 
+source /etc/overleaf/env.sh
+export LISTEN_ADDRESS=127.0.0.1
+
 exec /sbin/setuser www-data /usr/bin/node $NODE_PARAMS /overleaf/services/chat/app.js >> /var/log/overleaf/chat.log 2>&1

server-ce/runit/clsi-overleaf/run

@@ -15,4 +15,7 @@ if [ -e '/var/run/docker.sock' ]; then
   usermod -aG dockeronhost www-data
 fi
 
+source /etc/overleaf/env.sh
+export LISTEN_ADDRESS=127.0.0.1
+
 exec /sbin/setuser www-data /usr/bin/node $NODE_PARAMS /overleaf/services/clsi/app.js >> /var/log/overleaf/clsi.log 2>&1

server-ce/runit/contacts-overleaf/run

@@ -6,4 +6,7 @@ if [ "$DEBUG_NODE" == "true" ]; then
     NODE_PARAMS="--inspect=0.0.0.0:30360"
 fi
 
+source /etc/overleaf/env.sh
+export LISTEN_ADDRESS=127.0.0.1
+
 exec /sbin/setuser www-data /usr/bin/node $NODE_PARAMS /overleaf/services/contacts/app.js >> /var/log/overleaf/contacts.log 2>&1

server-ce/runit/docstore-overleaf/run

@@ -6,4 +6,7 @@ if [ "$DEBUG_NODE" == "true" ]; then
     NODE_PARAMS="--inspect=0.0.0.0:30160"
 fi
 
+source /etc/overleaf/env.sh
+export LISTEN_ADDRESS=127.0.0.1
+
 exec /sbin/setuser www-data /usr/bin/node $NODE_PARAMS /overleaf/services/docstore/app.js >> /var/log/overleaf/docstore.log 2>&1

server-ce/runit/document-updater-overleaf/run

@@ -6,4 +6,7 @@ if [ "$DEBUG_NODE" == "true" ]; then
     NODE_PARAMS="--inspect=0.0.0.0:30030"
 fi
 
+source /etc/overleaf/env.sh
+export LISTEN_ADDRESS=127.0.0.1
+
 exec /sbin/setuser www-data /usr/bin/node $NODE_PARAMS /overleaf/services/document-updater/app.js >> /var/log/overleaf/document-updater.log 2>&1

server-ce/runit/filestore-overleaf/run

@@ -1,2 +1,6 @@
 #!/bin/bash
+
+source /etc/overleaf/env.sh
+export LISTEN_ADDRESS=127.0.0.1
+
 exec /sbin/setuser www-data /usr/bin/node /overleaf/services/filestore/app.js >> /var/log/overleaf/filestore.log 2>&1

server-ce/runit/notifications-overleaf/run

@@ -6,4 +6,7 @@ if [ "$DEBUG_NODE" == "true" ]; then
     NODE_PARAMS="--inspect=0.0.0.0:30420"
 fi
 
+source /etc/overleaf/env.sh
+export LISTEN_ADDRESS=127.0.0.1
+
 exec /sbin/setuser www-data /usr/bin/node $NODE_PARAMS /overleaf/services/notifications/app.js >> /var/log/overleaf/notifications.log 2>&1

server-ce/runit/project-history-overleaf/run

@@ -6,4 +6,7 @@ if [ "$DEBUG_NODE" == "true" ]; then
     NODE_PARAMS="--inspect=0.0.0.0:30540"
 fi
 
+source /etc/overleaf/env.sh
+export LISTEN_ADDRESS=127.0.0.1
+
 exec /sbin/setuser www-data /usr/bin/node $NODE_PARAMS /overleaf/services/project-history/app.js >> /var/log/overleaf/project-history.log 2>&1

server-ce/runit/real-time-overleaf/run

@@ -1,2 +1,6 @@
 #!/bin/bash
+
+source /etc/overleaf/env.sh
+export LISTEN_ADDRESS=127.0.0.1
+
 exec /sbin/setuser www-data /usr/bin/node /overleaf/services/real-time/app.js >> /var/log/overleaf/real-time.log 2>&1

server-ce/runit/spelling-overleaf/run

@@ -6,4 +6,7 @@ if [ "$DEBUG_NODE" == "true" ]; then
     NODE_PARAMS="--inspect=0.0.0.0:30050"
 fi
 
+source /etc/overleaf/env.sh
+export LISTEN_ADDRESS=127.0.0.1
+
 exec /sbin/setuser www-data /usr/bin/node $NODE_PARAMS /overleaf/services/spelling/app.js >> /var/log/overleaf/spelling.log 2>&1

server-ce/runit/web-api-overleaf/run

@@ -6,6 +6,7 @@ if [ "$DEBUG_NODE" == "true" ]; then
     NODE_PARAMS="--inspect=0.0.0.0:30000"
 fi
 
+source /etc/overleaf/env.sh
 export LISTEN_ADDRESS=0.0.0.0
 export ENABLED_SERVICES="api"
 export METRICS_APP_NAME="web-api"

server-ce/runit/web-overleaf/run

@@ -6,6 +6,8 @@ if [ "$DEBUG_NODE" == "true" ]; then
     NODE_PARAMS="--inspect=0.0.0.0:40000"
 fi
 
+source /etc/overleaf/env.sh
+export LISTEN_ADDRESS=127.0.0.1
 export ENABLED_SERVICES="web"
 export WEB_PORT="4000"
 

services/document-updater/app/js/ProjectHistoryRedisManager.js

@@ -1,18 +1,17 @@
 const Settings = require('@overleaf/settings')
-const { promisifyAll } = require('@overleaf/promise-utils')
+const { callbackifyAll } = require('@overleaf/promise-utils')
 const projectHistoryKeys = Settings.redis?.project_history?.key_schema
 const rclient = require('@overleaf/redis-wrapper').createClient(
   Settings.redis.project_history
 )
 const logger = require('@overleaf/logger')
 const metrics = require('./Metrics')
 const { docIsTooLarge } = require('./Limits')
+const OError = require('@overleaf/o-error')
 
 const ProjectHistoryRedisManager = {
-  queueOps(projectId, ...rest) {
+  async queueOps(projectId, ...ops) {
     // Record metric for ops pushed onto queue
-    const callback = rest.pop()
-    const ops = rest
     for (const op of ops) {
       metrics.summary('redis.projectHistoryOps', op.length, { status: 'push' })
     }
@@ -30,24 +29,18 @@ const ProjectHistoryRedisManager = {
       }),
       Date.now()
     )
-    multi.exec(function (error, result) {
-      if (error) {
-        return callback(error)
-      }
-      // return the number of entries pushed onto the project history queue
-      callback(null, result[0])
-    })
+    const result = await multi.exec()
+    return result[0]
   },
 
-  queueRenameEntity(
+  async queueRenameEntity(
     projectId,
     projectHistoryId,
     entityType,
     entityId,
     userId,
     projectUpdate,
-    source,
-    callback
+    source
   ) {
     projectUpdate = {
       pathname: projectUpdate.pathname,
@@ -73,18 +66,17 @@ const ProjectHistoryRedisManager = {
     )
     const jsonUpdate = JSON.stringify(projectUpdate)
 
-    ProjectHistoryRedisManager.queueOps(projectId, jsonUpdate, callback)
+    return await ProjectHistoryRedisManager.queueOps(projectId, jsonUpdate)
   },
 
-  queueAddEntity(
+  async queueAddEntity(
     projectId,
     projectHistoryId,
     entityType,
     entityId,
     userId,
     projectUpdate,
-    source,
-    callback
+    source
   ) {
     projectUpdate = {
       pathname: projectUpdate.pathname,
@@ -111,16 +103,10 @@ const ProjectHistoryRedisManager = {
     )
     const jsonUpdate = JSON.stringify(projectUpdate)
 
-    ProjectHistoryRedisManager.queueOps(projectId, jsonUpdate, callback)
+    return await ProjectHistoryRedisManager.queueOps(projectId, jsonUpdate)
   },
 
-  queueResyncProjectStructure(
-    projectId,
-    projectHistoryId,
-    docs,
-    files,
-    callback
-  ) {
+  async queueResyncProjectStructure(projectId, projectHistoryId, docs, files) {
     logger.debug({ projectId, docs, files }, 'queue project structure resync')
     const projectUpdate = {
       resyncProjectStructure: { docs, files },
@@ -130,17 +116,16 @@ const ProjectHistoryRedisManager = {
       },
     }
     const jsonUpdate = JSON.stringify(projectUpdate)
-    ProjectHistoryRedisManager.queueOps(projectId, jsonUpdate, callback)
+    return await ProjectHistoryRedisManager.queueOps(projectId, jsonUpdate)
   },
 
-  queueResyncDocContent(
+  async queueResyncDocContent(
     projectId,
     projectHistoryId,
     docId,
     lines,
     version,
-    pathname,
-    callback
+    pathname
   ) {
     logger.debug(
       { projectId, docId, lines, version, pathname },
@@ -163,15 +148,16 @@ const ProjectHistoryRedisManager = {
     // project update length as an upper bound
     const sizeBound = jsonUpdate.length
     if (docIsTooLarge(sizeBound, lines, Settings.max_doc_length)) {
-      const err = new Error(
-        'blocking resync doc content insert into project history queue: doc is too large'
+      throw new OError(
+        'blocking resync doc content insert into project history queue: doc is too large',
+        { projectId, docId, docSize: sizeBound }
       )
-      logger.error({ projectId, docId, err, docSize: sizeBound }, err.message)
-      return callback(err)
     }
-    ProjectHistoryRedisManager.queueOps(projectId, jsonUpdate, callback)
+    return await ProjectHistoryRedisManager.queueOps(projectId, jsonUpdate)
   },
 }
 
-module.exports = ProjectHistoryRedisManager
-module.exports.promises = promisifyAll(ProjectHistoryRedisManager)
+module.exports = {
+  ...callbackifyAll(ProjectHistoryRedisManager),
+  promises: ProjectHistoryRedisManager,
+}