refactor(backend): remove deprecated GitHub sync endpoint from API spec

Author: Lasim

services/backend/api-spec.json

@@ -16976,30 +16976,6 @@
         }
       }
     },
-    "/api/mcp/github/sync/{serverId}": {
-      "post": {
-        "summary": "Sync MCP server from GitHub",
-        "tags": [
-          "MCP GitHub Integration"
-        ],
-        "description": "Sync an MCP server with its GitHub repository to update metadata and versions",
-        "parameters": [
-          {
-            "schema": {
-              "type": "string"
-            },
-            "in": "path",
-            "name": "serverId",
-            "required": true
-          }
-        ],
-        "responses": {
-          "200": {
-            "description": "Default Response"
-          }
-        }
-      }
-    },
     "/api/mcp/github/repo-info": {
       "get": {
         "summary": "Get GitHub repository info",

services/backend/api-spec.yaml

@@ -11983,22 +11983,6 @@ paths:
       responses:
         "200":
           description: Default Response
-  /api/mcp/github/sync/{serverId}:
-    post:
-      summary: Sync MCP server from GitHub
-      tags:
-        - MCP GitHub Integration
-      description: Sync an MCP server with its GitHub repository to update metadata
-        and versions
-      parameters:
-        - schema:
-            type: string
-          in: path
-          name: serverId
-          required: true
-      responses:
-        "200":
-          description: Default Response
   /api/mcp/github/repo-info:
     get:
       summary: Get GitHub repository info

services/backend/src/services/registrySyncService.ts

@@ -1,6 +1,8 @@
 import type { FastifyBaseLogger } from 'fastify';
 import type { AnyDatabase } from '../db';
 import { JobQueueService } from './jobQueueService';
+import { mcpServers } from '../db/schema.sqlite';
+import { inArray } from 'drizzle-orm';
 
 /**
  * Sync configuration options
@@ -64,18 +66,22 @@ export class RegistrySyncService {
     }, 'Starting MCP Registry sync via job queue');
 
     try {
-      // Fetch all servers from official registry
-      const allServers = await this.fetchAllServersFromRegistry(finalConfig.maxServers);
+      // Fetch servers from official registry with smart pagination
+      // If skipExisting + maxServers, fetch page-by-page and filter until we have enough
+      const serversToSync = await this.fetchNewServersFromRegistry(
+        finalConfig.maxServers,
+        finalConfig.skipExisting
+      );
 
       this.logger.info({
-        totalServers: allServers.length,
+        totalNewServers: serversToSync.length,
         maxServers: finalConfig.maxServers,
-      }, 'Fetched servers from official registry');
+      }, 'Collected new servers from official registry');
 
       // Create job batch for tracking
       const batch = await this.jobQueueService.createBatch(
         'mcp_registry_sync',
-        allServers.length,
+        serversToSync.length,
         {
           syncedBy,
           config: finalConfig,
@@ -85,8 +91,8 @@ export class RegistrySyncService {
 
       // Create individual jobs for each server
       let jobsCreated = 0;
-      for (let i = 0; i < allServers.length; i++) {
-        const server = allServers[i];
+      for (let i = 0; i < serversToSync.length; i++) {
+        const server = serversToSync[i];
         const scheduledFor = new Date(Date.now() + (i * finalConfig.rateLimitDelay * 1000));
 
         await this.jobQueueService.createJob(
@@ -101,7 +107,7 @@ export class RegistrySyncService {
             batchInfo: {
               batchId: batch.id,
               serverIndex: i + 1,
-              totalServers: allServers.length,
+              totalServers: serversToSync.length,
             },
           },
           {
@@ -114,21 +120,21 @@ export class RegistrySyncService {
       }
 
       const estimatedCompletion = new Date(
-        Date.now() + (allServers.length * finalConfig.rateLimitDelay * 1000)
+        Date.now() + (serversToSync.length * finalConfig.rateLimitDelay * 1000)
       );
 
       this.logger.info({
         operation: 'mcp_registry_sync_jobs_created',
         batchId: batch.id,
-        totalServers: allServers.length,
+        totalServers: serversToSync.length,
         jobsCreated,
         rateLimitDelay: finalConfig.rateLimitDelay,
         estimatedCompletion: estimatedCompletion.toISOString(),
       }, 'Created MCP server sync jobs');
 
       return {
         batchId: batch.id,
-        totalServers: allServers.length,
+        totalServers: serversToSync.length,
         jobsCreated,
         startTime: new Date(),
         estimatedCompletion,
@@ -141,7 +147,183 @@ export class RegistrySyncService {
   }
 
   /**
-   * Fetch all servers from the official registry
+   * Filter out servers that already exist in the database
+   * Queries the database for existing servers by official_name
+   * and returns only servers that don't exist yet
+   */
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  private async filterExistingServers(servers: any[]): Promise<any[]> {
+    if (servers.length === 0) return servers;
+    
+    // Extract official names from servers
+    const officialNames = servers
+      .map(s => s.name) // The 'name' field contains the official reverse-DNS name
+      .filter(Boolean);
+    
+    if (officialNames.length === 0) {
+      this.logger.warn('No official names found in servers, cannot filter existing servers');
+      return servers;
+    }
+    
+    this.logger.debug({
+      totalServers: servers.length,
+      officialNamesToCheck: officialNames.length,
+    }, 'Checking for existing servers in database by official_name');
+    
+    try {
+      // Query database for existing servers by official_name
+      const existing = await this.db
+        .select({ official_name: mcpServers.official_name })
+        .from(mcpServers)
+        .where(inArray(mcpServers.official_name, officialNames));
+      
+      // Build Set of existing names for fast lookup
+      const existingNames = new Set(
+        existing
+          // eslint-disable-next-line @typescript-eslint/no-explicit-any
+          .map((s: any) => s.official_name)
+          .filter(Boolean)
+      );
+      
+      this.logger.debug({
+        existingCount: existingNames.size,
+      }, 'Found existing servers in database');
+      
+      // Return only servers that don't exist
+      return servers.filter(s => {
+        const officialName = s.name;
+        return officialName && !existingNames.has(officialName);
+      });
+      
+    } catch (error) {
+      this.logger.error({ error }, 'Failed to query existing servers, continuing without filtering');
+      return servers;
+    }
+  }
+  
+  /**
+   * Fetch new servers from official registry with smart pagination
+   * 
+   * Strategy:
+   * 1. Fetch page 1 (50 servers)
+   * 2. Query DB for those 50 names only
+   * 3. Filter → get X new servers
+   * 4. If X >= maxServers → stop and return first maxServers
+   * 5. If X < maxServers → fetch page 2, repeat
+   * 
+   * This ensures we only fetch what we need and don't overload the database.
+   */
+   
+  private async fetchNewServersFromRegistry(
+    maxServers: number | null,
+    skipExisting: boolean
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  ): Promise<any[]> {
+    // If not skipping existing, use the old method (fetch all then filter)
+    if (!skipExisting) {
+      return this.fetchAllServersFromRegistry(maxServers);
+    }
+    
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const accumulatedNewServers: any[] = [];
+    let cursor: string | undefined;
+    let hasMore = true;
+    let pageNumber = 0;
+    
+    this.logger.debug({
+      maxServers,
+      skipExisting,
+      operation: 'smart_pagination_start'
+    }, 'Starting smart page-by-page fetch with filtering');
+    
+    while (hasMore) {
+      pageNumber++;
+      
+      // Check if we have enough new servers
+      if (maxServers && accumulatedNewServers.length >= maxServers) {
+        this.logger.debug({
+          pageNumber,
+          accumulatedCount: accumulatedNewServers.length,
+          maxServers,
+          operation: 'smart_pagination_complete'
+        }, 'Collected enough new servers, stopping pagination');
+        break;
+      }
+      
+      // Fetch one page (50 servers)
+      const batchResult = await this.fetchServersBatch(cursor, 50);
+      
+      if (!batchResult.success) {
+        throw new Error(`Failed to fetch servers: ${batchResult.error}`);
+      }
+      
+      const { servers, metadata } = batchResult.data;
+      
+      // Extract server data and filter for isLatest === true (CRITICAL: keep this filter!)
+      const serverData = servers
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        .filter((item: any) => {
+          const meta = item._meta?.['io.modelcontextprotocol.registry/official'];
+          return meta?.isLatest === true; // KEEP THIS AT ALL COSTS
+        })
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        .map((item: any) => {
+          const server = item.server || item;
+          return {
+            ...server,
+            _meta: item._meta
+          };
+        });
+      
+      this.logger.debug({
+        pageNumber,
+        rawBatchSize: servers.length,
+        afterIsLatestFilter: serverData.length,
+        cursor,
+      }, 'Fetched and filtered page from official registry');
+      
+      // Filter this page against database (only check these 50 servers)
+      const newServersInPage = await this.filterExistingServers(serverData);
+      
+      this.logger.debug({
+        pageNumber,
+        serversInPage: serverData.length,
+        newInPage: newServersInPage.length,
+        existingInPage: serverData.length - newServersInPage.length,
+        accumulatedSoFar: accumulatedNewServers.length,
+      }, 'Filtered page against database');
+      
+      // Add new servers from this page to accumulated list
+      accumulatedNewServers.push(...newServersInPage);
+      
+      // Check for next page
+      cursor = metadata.next_cursor;
+      hasMore = !!cursor;
+      
+      // Rate limiting for registry API
+      if (hasMore) {
+        await new Promise(resolve => setTimeout(resolve, 1000));
+      }
+    }
+    
+    // Return up to maxServers (or all if maxServers is null)
+    const result = maxServers 
+      ? accumulatedNewServers.slice(0, maxServers)
+      : accumulatedNewServers;
+    
+    this.logger.info({
+      totalPagesChecked: pageNumber,
+      totalNewServersFound: accumulatedNewServers.length,
+      returningCount: result.length,
+      maxServers,
+      operation: 'smart_pagination_result'
+    }, 'Smart pagination completed');
+    
+    return result;
+  }
+  
+  /**
+   * Fetch all servers from the official registry (fallback method)
    */
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
   private async fetchAllServersFromRegistry(maxServers: number | null): Promise<any[]> {
@@ -171,10 +353,12 @@ export class RegistrySyncService {
       // We need to attach _meta to the server object so it's available for metadata extraction
       // FILTER: Only include servers where _meta.io.modelcontextprotocol.registry/official.isLatest === true
       const serverData = servers
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
         .filter((item: any) => {
           const meta = item._meta?.['io.modelcontextprotocol.registry/official'];
           return meta?.isLatest === true;
         })
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
         .map((item: any) => {
           const server = item.server || item;
           // Attach _meta to the server object for later extraction