feat: add server-side pagination to API endpoints

cypress/e2e/repo.cy.js

@@ -176,4 +176,48 @@ describe('Repo', () => {
       });
     });
   });
+
+  describe('Pagination and search', () => {
+    beforeEach(() => {
+      cy.login('admin', 'admin');
+      cy.visit('/dashboard/repo');
+    });
+
+    it('sends search param to API when typing in search box', () => {
+      cy.intercept('GET', '**/api/v1/repo*').as('initialLoad');
+      cy.wait('@initialLoad');
+
+      cy.intercept('GET', '**/api/v1/repo*').as('searchRequest');
+      cy.get('input[type="text"]').first().type('finos');
+
+      cy.wait('@searchRequest').its('request.url').should('include', 'search=finos');
+    });
+
+    it('sends page=2 to API when clicking Next', () => {
+      cy.intercept('GET', '**/api/v1/repo*').as('getRepos');
+
+      cy.wait('@getRepos');
+
+      cy.contains('button', 'Next').then(($btn) => {
+        if (!$btn.is(':disabled')) {
+          cy.intercept('GET', '**/api/v1/repo*').as('getReposPage2');
+          cy.contains('button', 'Next').click();
+          cy.wait('@getReposPage2').its('request.url').should('include', 'page=2');
+        }
+      });
+    });
+
+    it('sends updated limit to API when changing items per page', () => {
+      cy.intercept('GET', '**/api/v1/repo*').as('getRepos');
+
+      cy.wait('@getRepos');
+
+      cy.intercept('GET', '**/api/v1/repo*').as('getReposLimited');
+
+      cy.get('.paginationContainer').find('.MuiSelect-root').click();
+      cy.get('[data-value="25"]').click();
+
+      cy.wait('@getReposLimited').its('request.url').should('include', 'limit=25');
+    });
+  });
 });

cypress/support/commands.js

@@ -134,18 +134,17 @@ Cypress.Commands.add('getTestRepoId', () => {
         `GET ${url} returned status ${res.status}: ${JSON.stringify(res.body).slice(0, 500)}`,
       );
     }
-    if (!Array.isArray(res.body)) {
+    const repos = res.body.repos ?? res.body.data ?? res.body;
+    if (!Array.isArray(repos)) {
       throw new Error(
-        `GET ${url} returned non-array (${typeof res.body}): ${JSON.stringify(res.body).slice(0, 500)}`,
+        `GET ${url} returned unexpected shape: ${JSON.stringify(res.body).slice(0, 500)}`,
       );
     }
     const gitServerTarget = Cypress.env('GIT_SERVER_TARGET') || 'git-server:8443';
-    const repo = res.body.find(
-      (r) => r.url === `https://${gitServerTarget}/test-owner/test-repo.git`,
-    );
+    const repo = repos.find((r) => r.url === `https://${gitServerTarget}/test-owner/test-repo.git`);
     if (!repo) {
       throw new Error(
-        `test-owner/test-repo not found in database. Repos: ${res.body.map((r) => r.url).join(', ')}`,
+        `test-owner/test-repo not found in database. Repos: ${repos.map((r) => r.url).join(', ')}`,
       );
     }
     return cy.wrap(repo._id);
@@ -159,8 +158,9 @@ Cypress.Commands.add('cleanupTestRepos', () => {
       url: `${getApiBaseUrl()}/api/v1/repo`,
       failOnStatusCode: false,
     }).then((res) => {
-      if (res.status !== 200 || !Array.isArray(res.body)) return;
-      const testRepos = res.body.filter((r) => r.project === 'cypress-test');
+      const repos = res.body.repos ?? res.body.data ?? res.body;
+      if (res.status !== 200 || !Array.isArray(repos)) return;
+      const testRepos = repos.filter((r) => r.project === 'cypress-test');
       testRepos.forEach((repo) => {
         cy.request({
           method: 'DELETE',

packages/git-proxy-cli/index.ts

@@ -106,12 +106,15 @@ async function getGitPushes(filters: Partial<PushQuery>) {
 
   try {
     const cookies = JSON.parse(fs.readFileSync(GIT_PROXY_COOKIE_FILE, 'utf8'));
-    const { data } = await axios.get<Action[]>(`${baseUrl}/api/v1/push/`, {
-      headers: { Cookie: cookies },
-      params: filters,
-    });
+    const response = await axios.get<{ pushes: Action[]; total: number }>(
+      `${baseUrl}/api/v1/push/`,
+      {
+        headers: { Cookie: cookies },
+        params: filters,
+      },
+    );
 
-    const records = data.map((push: Action) => {
+    const records = response.data.pushes.map((push: Action) => {
       const {
         id,
         repo,

src/db/file/helper.ts

@@ -15,8 +15,42 @@
  */
 
 import { existsSync, mkdirSync } from 'fs';
+import Datastore from '@seald-io/nedb';
+import { PaginatedResult } from '../types';
 
 export const getSessionStore = (): undefined => undefined;
 export const initializeFolders = () => {
   if (!existsSync('./.data/db')) mkdirSync('./.data/db', { recursive: true });
 };
+
+export const paginatedFind = <T>(
+  db: Datastore,
+  filter: Record<string, unknown>,
+  sort: Record<string, 1 | -1>,
+  skip: number,
+  limit: number,
+): Promise<PaginatedResult<T>> => {
+  const countPromise = new Promise<number>((resolve, reject) => {
+    db.count(filter as any, (err: Error | null, count: number) => {
+      /* istanbul ignore if */
+      if (err) reject(err);
+      else resolve(count);
+    });
+  });
+
+  const dataPromise = new Promise<T[]>((resolve, reject) => {
+    db.find(filter as any)
+      .sort(sort)
+      .skip(skip)
+      .limit(limit)
+      .exec((err: Error | null, docs: any[]) => {
+        /* istanbul ignore if */
+        if (err) reject(err);
+        else resolve(docs);
+      });
+  });
+
+  return limit > 0
+    ? Promise.all([dataPromise, countPromise]).then(([data, total]) => ({ data, total }))
+    : dataPromise.then((data) => ({ data, total: data.length }));
+};

src/db/file/pushes.ts

@@ -17,8 +17,9 @@
 import _ from 'lodash';
 import Datastore from '@seald-io/nedb';
 import { Action } from '../../proxy/actions/Action';
-import { toClass } from '../helper';
-import { PushQuery } from '../types';
+import { toClass, buildSearchFilter, buildSort } from '../helper';
+import { paginatedFind } from './helper';
+import { PaginatedResult, PaginationOptions, PushQuery } from '../types';
 import { CompletedAttestation, Rejection } from '../../proxy/processors/types';
 import { handleErrorAndLog } from '../../utils/errors';
 
@@ -49,25 +50,33 @@ const defaultPushQuery: Partial<PushQuery> = {
   type: 'push',
 };
 
-export const getPushes = (query: Partial<PushQuery>): Promise<Action[]> => {
+export const getPushes = (
+  query: Partial<PushQuery>,
+  pagination?: PaginationOptions,
+): Promise<PaginatedResult<Action>> => {
   if (!query) query = defaultPushQuery;
-  return new Promise((resolve, reject) => {
-    db.find(query)
-      .sort({ timestamp: -1 })
-      .exec((err, docs) => {
-        // ignore for code coverage as neDB rarely returns errors even for an invalid query
-        /* istanbul ignore if */
-        if (err) {
-          reject(err);
-        } else {
-          resolve(
-            _.chain(docs)
-              .map((x) => toClass(x, Action.prototype))
-              .value(),
-          );
-        }
-      });
-  });
+
+  const baseQuery = buildSearchFilter(
+    { ...query },
+    ['repo', 'branch', 'commitTo', 'user'],
+    pagination?.search,
+  );
+  const sort = buildSort(pagination, 'timestamp', -1, [
+    'timestamp',
+    'repo',
+    'branch',
+    'commitTo',
+    'user',
+  ]);
+  const skip = pagination?.skip ?? 0;
+  const limit = pagination?.limit ?? 0;
+
+  return paginatedFind<Action>(db, baseQuery, sort, skip, limit).then(({ data, total }) => ({
+    data: _.chain(data)
+      .map((x) => toClass(x, Action.prototype))
+      .value(),
+    total,
+  }));
 };
 
 export const getPush = async (id: string): Promise<Action | null> => {
@@ -157,5 +166,5 @@ export const cancel = async (id: string): Promise<{ message: string }> => {
   action.canceled = true;
   action.rejected = false;
   await writeAudit(action);
-  return { message: `cancel ${id}` };
+  return { message: `canceled ${id}` };
 };

src/db/file/repo.ts

@@ -17,8 +17,9 @@
 import Datastore from '@seald-io/nedb';
 import _ from 'lodash';
 
-import { Repo, RepoQuery } from '../types';
-import { toClass } from '../helper';
+import { PaginatedResult, PaginationOptions, Repo, RepoQuery } from '../types';
+import { toClass, buildSearchFilter, buildSort } from '../helper';
+import { paginatedFind } from './helper';
 import { handleErrorAndLog } from '../../utils/errors';
 
 const COMPACTION_INTERVAL = 1000 * 60 * 60 * 24; // once per day
@@ -42,25 +43,25 @@ try {
 db.ensureIndex({ fieldName: 'name', unique: false });
 db.setAutocompactionInterval(COMPACTION_INTERVAL);
 
-export const getRepos = async (query: Partial<RepoQuery> = {}): Promise<Repo[]> => {
+export const getRepos = async (
+  query: Partial<RepoQuery> = {},
+  pagination?: PaginationOptions,
+): Promise<PaginatedResult<Repo>> => {
   if (query?.name) {
     query.name = query.name.toLowerCase();
   }
-  return new Promise<Repo[]>((resolve, reject) => {
-    db.find(query, (err: Error, docs: Repo[]) => {
-      // ignore for code coverage as neDB rarely returns errors even for an invalid query
-      /* istanbul ignore if */
-      if (err) {
-        reject(err);
-      } else {
-        resolve(
-          _.chain(docs)
-            .map((x) => toClass(x, Repo.prototype))
-            .value(),
-        );
-      }
-    });
-  });
+
+  const baseQuery = buildSearchFilter({ ...query }, ['name', 'project', 'url'], pagination?.search);
+  const sort = buildSort(pagination, 'name', 1, ['name', 'project', 'url']);
+  const skip = pagination?.skip ?? 0;
+  const limit = pagination?.limit ?? 0;
+
+  return paginatedFind<Repo>(db, baseQuery, sort, skip, limit).then(({ data, total }) => ({
+    data: _.chain(data)
+      .map((x) => toClass(x, Repo.prototype))
+      .value(),
+    total,
+  }));
 };
 
 export const getRepo = async (name: string): Promise<Repo | null> => {

src/db/file/users.ts

@@ -17,8 +17,10 @@
 import fs from 'fs';
 import Datastore from '@seald-io/nedb';
 
-import { User, UserQuery } from '../types';
+import { PaginatedResult, PaginationOptions, User, UserQuery } from '../types';
 import { handleErrorAndLog } from '../../utils/errors';
+import { buildSearchFilter, buildSort } from '../helper';
+import { paginatedFind } from './helper';
 
 const COMPACTION_INTERVAL = 1000 * 60 * 60 * 24; // once per day
 
@@ -180,22 +182,30 @@ export const updateUser = (user: Partial<User>): Promise<void> => {
   });
 };
 
-export const getUsers = (query: Partial<UserQuery> = {}): Promise<User[]> => {
+export const getUsers = (
+  query: Partial<UserQuery> = {},
+  pagination?: PaginationOptions,
+): Promise<PaginatedResult<User>> => {
   if (query.username) {
     query.username = query.username.toLowerCase();
   }
   if (query.email) {
     query.email = query.email.toLowerCase();
   }
-  return new Promise<User[]>((resolve, reject) => {
-    db.find(query, (err: Error, docs: User[]) => {
-      // ignore for code coverage as neDB rarely returns errors even for an invalid query
-      /* istanbul ignore if */
-      if (err) {
-        reject(err);
-      } else {
-        resolve(docs);
-      }
-    });
-  });
+
+  const baseQuery = buildSearchFilter(
+    { ...query },
+    ['username', 'displayName', 'email', 'gitAccount'],
+    pagination?.search,
+  );
+  const sort = buildSort(pagination, 'username', 1, [
+    'username',
+    'displayName',
+    'email',
+    'gitAccount',
+  ]);
+  const skip = pagination?.skip ?? 0;
+  const limit = pagination?.limit ?? 0;
+
+  return paginatedFind<User>(db, baseQuery, sort, skip, limit);
 };

src/db/helper.ts

@@ -14,6 +14,36 @@
  * limitations under the License.
  */
 
+import { PaginationOptions } from './types';
+
+export const escapeRegex = (s: string): string => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+
+export const buildSearchFilter = (
+  baseQuery: Record<string, unknown>,
+  searchFields: string[],
+  search?: string,
+): Record<string, unknown> => {
+  if (!search) return baseQuery;
+  const regex = new RegExp(escapeRegex(search), 'i');
+  return { ...baseQuery, $or: searchFields.map((f) => ({ [f]: regex })) };
+};
+
+export const buildSort = (
+  pagination: PaginationOptions | undefined,
+  defaultField: string,
+  defaultDir: 1 | -1,
+  allowedFields?: string[],
+): Record<string, 1 | -1> => {
+  const requestedField = pagination?.sortBy;
+  const field =
+    requestedField && (!allowedFields || allowedFields.includes(requestedField))
+      ? requestedField
+      : defaultField;
+  const dir =
+    pagination?.sortOrder === 'asc' ? 1 : pagination?.sortOrder === 'desc' ? -1 : defaultDir;
+  return { [field]: dir };
+};
+
 export const toClass = function <T, U>(obj: T, proto: U): U {
   const out = JSON.parse(JSON.stringify(obj));
   out.__proto__ = proto;