feat(#84): separate request to get pending, and null if unknown

medic · Jul 31, 2024 · 31c01a4 · 31c01a4
1 parent e676313
commit 31c01a4
Show file tree

Hide file tree

Showing 4 changed files with 109 additions and 17 deletions.
diff --git a/couch2pg/package-lock.json b/couch2pg/package-lock.json
diff --git a/couch2pg/package.json b/couch2pg/package.json
@@ -13,6 +13,7 @@
   "author": "",
   "license": "ISC",
   "dependencies": {
+    "axios": "^1.7.2",
     "dotenv": "^16.4.5",
     "pg": "^8.11.5",
     "pouchdb-adapter-http": "^8.0.1",

diff --git a/couch2pg/src/importer.js b/couch2pg/src/importer.js
@@ -1,6 +1,7 @@
 const BATCH_SIZE = process.env.BATCH_SIZE || 1000;
 
 import * as db from './db.js';
+import axios from 'axios';
 
 const SELECT_SEQ_STMT = `SELECT seq FROM ${db.postgresProgressTable} WHERE source = $1`;
 const INSERT_SEQ_STMT = `
@@ -41,10 +42,6 @@ const removeSecurityDetails = (doc) => {
   }
 };
 
-const getNumberFromSeq = (seq) => {
-  return Number(seq.split('-')[0]);
-};
-
 const getSeq = async (source) => {
   const client = await db.getPgClient();
   const result = await client.query(SELECT_SEQ_STMT, [source]);
@@ -142,7 +139,7 @@ const importChangesBatch = async (couchDb, source) => {
     pending = await getPending(couchDb, seq);
   } catch (error) {
     console.error('Error getting pending:', error);
-    pending = 0;
+    pending = null;
   }
 
   const changes = await couchDb.changes({ limit: BATCH_SIZE, since: seq, seq_interval: BATCH_SIZE });
@@ -164,8 +161,11 @@ const importChangesBatch = async (couchDb, source) => {
 };
 
 const getPending = async (couchDb, seq) => {
-  const info = await couchDb.info();
-  return getNumberFromSeq(info.update_seq) - getNumberFromSeq(seq);
+  const res = await axios.get(`${couchDb.name}/_changes?limit=0&since=${seq}`);
+  if (res.status === 200 && res.data?.pending) {
+    return res.data.pending;
+  }
+  return null;
 };
 
 export default async (couchdb) => {

diff --git a/couch2pg/tests/unit/importer.spec.js b/couch2pg/tests/unit/importer.spec.js
@@ -77,7 +77,7 @@ describe('importer', () => {
 
     expect(pgClient.query.calledTwice).to.equal(true);
     expect(pgClient.query.args[0]).to.deep.equal([getSeqMatch(), ['thehost/medic']]);
-    expect(pgClient.query.args[1]).to.deep.equal([updateSeqMatch(), ['21-vvv', 0, 'thehost/medic']]);
+    expect(pgClient.query.args[1]).to.deep.equal([updateSeqMatch(), ['21-vvv', null, 'thehost/medic']]);
   });
 
   it('should start with 0 seq if no checkpointer is found', async () => {
@@ -94,7 +94,7 @@ describe('importer', () => {
     expect(pgClient.query.calledThrice).to.equal(true);
     expect(pgClient.query.args[0]).to.deep.equal([getSeqMatch(), ['host/db']]);
     expect(pgClient.query.args[1]).to.deep.equal([insertSeqMatch(), [0, null, 'host/db']]);
-    expect(pgClient.query.args[2]).to.deep.equal([updateSeqMatch(), ['73-1', 0, 'host/db']]);
+    expect(pgClient.query.args[2]).to.deep.equal([updateSeqMatch(), ['73-1', null, 'host/db']]);
   });
 
   it('should start with checkpointer seq when found', async () => {
@@ -135,8 +135,8 @@ describe('importer', () => {
 
     expect(seqQueries.update.calledTwice).to.equal(true);
     expect(seqQueries.update.args).to.deep.equal([
-      [updateSeqMatch(), ['23-ppp', 0, 'thehost/medic']],
-      [updateSeqMatch(), ['25-vvv', 0, 'thehost/medic']],
+      [updateSeqMatch(), ['23-ppp', null, 'thehost/medic']],
+      [updateSeqMatch(), ['25-vvv', null, 'thehost/medic']],
     ]);
 
     expect(couchDb.allDocs.calledOnce).to.equal(true);
@@ -206,10 +206,10 @@ describe('importer', () => {
 
     expect(seqQueries.update.callCount).to.equal(4);
     expect(seqQueries.update.args).to.deep.equal([
-      [updateSeqMatch(), ['3-seq', 0, 'thehost/medic']],
-      [updateSeqMatch(), ['6-seq', 0, 'thehost/medic']],
-      [updateSeqMatch(), ['9-seq', 0, 'thehost/medic']],
-      [updateSeqMatch(), ['9-seq', 0, 'thehost/medic']],
+      [updateSeqMatch(), ['3-seq', null, 'thehost/medic']],
+      [updateSeqMatch(), ['6-seq', null, 'thehost/medic']],
+      [updateSeqMatch(), ['9-seq', null, 'thehost/medic']],
+      [updateSeqMatch(), ['9-seq', null, 'thehost/medic']],
     ]);
 
     expect(couchDb.allDocs.callCount).to.equal(3);
@@ -500,8 +500,8 @@ describe('importer', () => {
 
     expect(seqQueries.update.calledTwice).to.equal(true);
     expect(seqQueries.update.args).to.deep.equal([
-      [updateSeqMatch(), ['23-ppp', 0, 'thehost/medic']],
-      [updateSeqMatch(), ['25-vvv', 0, 'thehost/medic']],
+      [updateSeqMatch(), ['23-ppp', null, 'thehost/medic']],
+      [updateSeqMatch(), ['25-vvv', null, 'thehost/medic']],
     ]);
 
     expect(couchDb.allDocs.calledOnce).to.equal(true);