mcp-agents-ai · ups216 · Apr 29, 2025 · Apr 29, 2025 · Apr 29, 2025 · Apr 29, 2025
diff --git a/server/package.json b/server/package.json
@@ -12,11 +12,11 @@
     "test:integration": "vitest run tests/integration/*.test.ts",
     "test:mock": "vitest run tests/mock/*.test.ts",
     "crawl-servers": "tsx src/data/mcp_servers_crawler.ts --url https://raw.githubusercontent.com/modelcontextprotocol/servers/refs/heads/main/README.md",
-    "update-server-types": "tsx src/data/mcp_server_crawler_result_updater.ts",
+    "crawl-servers-postprocess": "tsx src/data/mcp_server_crawler_result_updater.ts",
     "clean-duplicates": "tsx src/data/clean_duplicate.ts",
     "process_categories": "tsx src/data/process_categories.ts",
     "process_locales": "tsx src/data/process_locales.ts",
-    "process_githubinfo": "tsx src/data/process_githubinfo.ts"
+    "process_githubinfo": "tsx src/data/process_githubinfo.ts --batch_size 200"
   },
   "dependencies": {
     "axios": "^1.6.0",

diff --git a/server/src/data/mcp_servers_official_list.json b/server/src/data/mcp_servers_official_list.json
@@ -1,7 +1,7 @@
 {
   "metadata": {
     "totalServers": 444,
-    "extractedAt": "2025-04-27T15:31:45.593Z",
+    "extractedAt": "2025-04-29T04:06:45.722Z",
     "sourceUrl": "https://raw.githubusercontent.com/modelcontextprotocol/servers/refs/heads/main/README.md",
     "baseRepoUrl": "https://github.com/modelcontextprotocol/servers",
     "defaultBranch": "main",

diff --git a/server/src/data/process_githubinfo.log.json b/server/src/data/process_githubinfo.log.json
@@ -1,5 +1,5 @@
 {
-  "lastProcessed": "2025-04-28T08:45:46.926Z",
+  "lastProcessed": "2025-04-29T07:41:12.835Z",
   "processedFiles": [
     "0006b282-ac88-4c32-b76c-02476e972a04_githubprojects",
     "003f3571-5f97-4f84-b126-f09b89e4247e_amadeus",

diff --git a/server/src/data/process_githubinfo.ts b/server/src/data/process_githubinfo.ts
@@ -8,6 +8,23 @@ import path from 'path';
 import { fileURLToPath } from 'url';
 import { fetchGithubInfo, extractGithubRepoInfo } from '../lib/githubEnrichment.js';
 
+// Parse command line arguments
+const args = process.argv.slice(2);
+let BATCH_SIZE: number | null = null; // Null means process all records
+
+// Process command line arguments
+for (let i = 0; i < args.length; i++) {
+  if (args[i] === '--batch_size' && i + 1 < args.length) {
+    const batchSize = parseInt(args[i + 1], 10);
+    if (!isNaN(batchSize) && batchSize > 0) {
+      BATCH_SIZE = batchSize;
+      i++; // Skip the next argument as it's the value
+    } else {
+      console.error(`Invalid batch size: ${args[i + 1]}. Will process all records.`);
+    }
+  }
+}
+
 // Get the directory name in ESM
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
@@ -34,11 +51,23 @@ function ensureDirectoryExists(dirPath: string): void {
 }
 
 // Function to load the log file or create it if it doesn't exist
-function loadProcessedLog(): ProcessedLog {
+function loadProcessedLog(allFilesCount: number): ProcessedLog {
   if (fs.existsSync(LOG_FILE)) {
     try {
       const logContent = fs.readFileSync(LOG_FILE, 'utf8');
-      return JSON.parse(logContent) as ProcessedLog;
+      const logData = JSON.parse(logContent) as ProcessedLog;
+
+      // Check if we've already processed all files and should start fresh
+      if (logData.processedFiles.length >= allFilesCount) {
+        console.log(`Log file shows all ${logData.processedFiles.length} files already processed. Starting fresh.`);
+        return {
+          lastProcessed: new Date().toISOString(),
+          processedFiles: [],
+          errors: {}
+        };
+      }
+
+      return logData;
     } catch (error) {
       console.warn(`Error reading log file, creating a new one: ${error}`);
     }
@@ -153,21 +182,22 @@ async function updateGithubInfoInFile(filePath: string): Promise<boolean> {
 async function processAllFiles(): Promise<void> {
   console.log('Starting GitHub info update process...');
   console.log(`Looking for JSON files in: ${SPLIT_DIR}`);
+  console.log(BATCH_SIZE ? `Batch size set to: ${BATCH_SIZE}` : `Processing all remaining records`);
+
+  // Get all JSON files from split directory (only in the root, not in language subdirectories)
+  const allFiles = fs.readdirSync(SPLIT_DIR)
+    .filter(file => file.endsWith('.json') && fs.statSync(path.join(SPLIT_DIR, file)).isFile());
+
+  console.log(`Found ${allFiles.length} total JSON files in root directory`);
 
   // Load processed log
-  const processedLog = loadProcessedLog();
+  const processedLog = loadProcessedLog(allFiles.length);
   console.log(`Loaded processing log. Last run: ${processedLog.lastProcessed}`);
   console.log(`Previously processed ${processedLog.processedFiles.length} files`);
 
   // Setup handlers to save progress on interruption
   setupShutdownHandlers(processedLog);
 
-  // Get all JSON files from split directory (only in the root, not in language subdirectories)
-  const allFiles = fs.readdirSync(SPLIT_DIR)
-    .filter(file => file.endsWith('.json') && fs.statSync(path.join(SPLIT_DIR, file)).isFile());
-
-  console.log(`Found ${allFiles.length} total JSON files in root directory`);
-
   // Filter out already processed files
   const filesToProcess = allFiles.filter(file => {
     const hubId = getHubIdFromFilename(file);
@@ -181,11 +211,17 @@ async function processAllFiles(): Promise<void> {
     return;
   }
 
-  // Process each file
-  for (const [index, file] of filesToProcess.entries()) {
+  // Limit the number of files to process based on batch size if provided
+  const filesToProcessInThisBatch = BATCH_SIZE ? filesToProcess.slice(0, BATCH_SIZE) : filesToProcess;
+  console.log(BATCH_SIZE 
+    ? `Processing batch of ${filesToProcessInThisBatch.length} files (limited by batch size ${BATCH_SIZE})`
+    : `Processing all ${filesToProcessInThisBatch.length} remaining files`);
+
+  // Process each file in the batch
+  for (const [index, file] of filesToProcessInThisBatch.entries()) {
     try {
       const hubId = getHubIdFromFilename(file);
-      console.log(`Processing file ${index + 1}/${filesToProcess.length}: ${file} (hubId: ${hubId})`);
+      console.log(`Processing file ${index + 1}/${filesToProcess.length} ${BATCH_SIZE ? `(batch_size: ${BATCH_SIZE})` : ''}: ${file} (hubId: ${hubId})`);
       const filePath = path.join(SPLIT_DIR, file);
 
       // Update GitHub info in the main file
@@ -227,6 +263,18 @@ async function processAllFiles(): Promise<void> {
   } else {
     console.log('GitHub info update process completed successfully!');
   }
+
+  // Report on overall progress
+  console.log(`Processed ${filesToProcessInThisBatch.length} files ${BATCH_SIZE ? 'in this batch' : ''}.`);
+  console.log(`Total progress: ${processedLog.processedFiles.length}/${allFiles.length} files processed.`);
+
+  if (processedLog.processedFiles.length < allFiles.length) {
+    console.log(BATCH_SIZE 
+      ? `Run the script again to process the next batch.` 
+      : `Some files may have been skipped due to errors. Check the log file for details.`);
+  } else {
+    console.log(`All files have been processed. The log file will be reset on next run.`);
+  }
 }
 
 // Execute the main function

diff --git a/server/src/data/split/00739d70-842d-4d36-bb98-04bde51a2220_kubernetes.json b/server/src/data/split/00739d70-842d-4d36-bb98-04bde51a2220_kubernetes.json
@@ -16,7 +16,7 @@
   ],
   "requiresApiKey": false,
   "isRecommended": false,
-  "githubStars": 493,
+  "githubStars": 502,
   "downloadCount": 414,
   "createdAt": "2025-02-17T22:30:26.383193Z",
   "updatedAt": "2025-04-22T03:59:36Z",

diff --git a/server/src/data/split/0092c98a-a48a-41fd-bdc8-e69c66e207a8_youtubesubtitles.json b/server/src/data/split/0092c98a-a48a-41fd-bdc8-e69c66e207a8_youtubesubtitles.json
@@ -16,7 +16,7 @@
   ],
   "requiresApiKey": false,
   "isRecommended": true,
-  "githubStars": 309,
+  "githubStars": 310,
   "downloadCount": 912,
   "createdAt": "2025-02-17T22:27:37.384353Z",
   "updatedAt": "2025-03-20T17:17:38Z",

diff --git a/server/src/data/split/00cca75b-f426-4279-ae1c-7e6b80636b50_youtubetranscripts.json b/server/src/data/split/00cca75b-f426-4279-ae1c-7e6b80636b50_youtubetranscripts.json
@@ -16,7 +16,7 @@
   ],
   "requiresApiKey": false,
   "isRecommended": false,
-  "githubStars": 169,
+  "githubStars": 170,
   "downloadCount": 0,
   "createdAt": "2025-03-17T08:29:20.399027+00:00",
   "updatedAt": "2025-03-10T05:10:04Z",
@@ -25,6 +25,6 @@
   "isReferenceServer": false,
   "isCommunityServer": true,
   "githubLatestCommit": "fe3b6a8808116a5d08adfe9197f8391a6e359e77",
-  "githubForks": 24,
+  "githubForks": 25,
   "licenseType": "MIT"
 }
diff --git a/server/src/data/split/00f1ad7c-0109-47d9-893d-004fe0b555f7_agentcarefhiremr.json b/server/src/data/split/00f1ad7c-0109-47d9-893d-004fe0b555f7_agentcarefhiremr.json
@@ -16,7 +16,7 @@
   ],
   "requiresApiKey": false,
   "isRecommended": false,
-  "githubStars": 39,
+  "githubStars": 40,
   "downloadCount": 0,
   "createdAt": "2025-03-17T08:29:23.366219+00:00",
   "updatedAt": "2025-03-13T05:56:46Z",

diff --git a/server/src/data/split/022d2a79-e4da-47b2-a138-2560f3822400_elevenlabstexttospeech.json b/server/src/data/split/022d2a79-e4da-47b2-a138-2560f3822400_elevenlabstexttospeech.json
@@ -25,6 +25,6 @@
   "isReferenceServer": false,
   "isCommunityServer": true,
   "githubLatestCommit": "ba2d6d946f0334305f8a888d47338f55222f4af1",
-  "githubForks": 1,
+  "githubForks": 2,
   "licenseType": null
 }
diff --git a/server/src/data/split/0272fd9a-3249-4990-bed5-d66aaa4ebdb4_browseruse.json b/server/src/data/split/0272fd9a-3249-4990-bed5-d66aaa4ebdb4_browseruse.json
@@ -16,7 +16,7 @@
   ],
   "requiresApiKey": false,
   "isRecommended": false,
-  "githubStars": 18,
+  "githubStars": 17,
   "downloadCount": 0,
   "createdAt": "2025-03-17T08:29:23.366219+00:00",
   "updatedAt": "2025-04-27T21:48:21Z",

diff --git a/server/src/data/split/02f9e25f-4904-4350-8a2e-a6a0a69f9dfb_onenote.json b/server/src/data/split/02f9e25f-4904-4350-8a2e-a6a0a69f9dfb_onenote.json
@@ -9,7 +9,7 @@
   "tags": [],
   "requiresApiKey": false,
   "isRecommended": false,
-  "githubStars": 2,
+  "githubStars": 3,
   "downloadCount": 0,
   "createdAt": "2025-04-27T14:08:34.434Z",
   "updatedAt": "2025-04-28T00:48:02Z",

diff --git a/server/src/data/split/03490ac9-ce58-485d-9166-a295173de417_fetch.json b/server/src/data/split/03490ac9-ce58-485d-9166-a295173de417_fetch.json
@@ -16,7 +16,7 @@
   ],
   "requiresApiKey": false,
   "isRecommended": false,
-  "githubStars": 41714,
+  "githubStars": 42020,
   "downloadCount": 0,
   "createdAt": "2025-03-17T08:29:19.654593+00:00",
   "updatedAt": "2025-04-27T13:54:22Z",
@@ -25,6 +25,6 @@
   "isReferenceServer": false,
   "isCommunityServer": true,
   "githubLatestCommit": "de1abc85a7ddbe408fffc00f783c7e9f1a69b6b3",
-  "githubForks": 4574,
+  "githubForks": 4612,
   "licenseType": "MIT"
 }
diff --git a/server/src/data/split/04373d12-e650-4b53-b91b-4945607391e2_postgrest.json b/server/src/data/split/04373d12-e650-4b53-b91b-4945607391e2_postgrest.json
@@ -16,7 +16,7 @@
   ],
   "requiresApiKey": false,
   "isRecommended": true,
-  "githubStars": 1297,
+  "githubStars": 1303,
   "downloadCount": 390,
   "createdAt": "2025-02-17T22:27:20.529942Z",
   "updatedAt": "2025-04-25T21:14:26Z",

diff --git a/server/src/data/split/045d532b-d1a6-41f0-a836-b8c40e2caacc_awsknowledgebase.json b/server/src/data/split/045d532b-d1a6-41f0-a836-b8c40e2caacc_awsknowledgebase.json
@@ -16,7 +16,7 @@
   ],
   "requiresApiKey": false,
   "isRecommended": true,
-  "githubStars": 41714,
+  "githubStars": 42020,
   "downloadCount": 1086,
   "createdAt": "2025-02-18T05:44:51.508867Z",
   "updatedAt": "2025-04-27T13:54:22Z",
@@ -25,6 +25,6 @@
   "isReferenceServer": true,
   "isCommunityServer": false,
   "githubLatestCommit": "de1abc85a7ddbe408fffc00f783c7e9f1a69b6b3",
-  "githubForks": 4574,
+  "githubForks": 4612,
   "licenseType": "MIT"
 }
diff --git a/server/src/data/split/05460e9c-6aab-4f63-9e94-0c5c0f61bc78_inboxzero.json b/server/src/data/split/05460e9c-6aab-4f63-9e94-0c5c0f61bc78_inboxzero.json
@@ -16,7 +16,7 @@
   ],
   "requiresApiKey": false,
   "isRecommended": true,
-  "githubStars": 7101,
+  "githubStars": 7119,
   "downloadCount": 0,
   "createdAt": "2025-03-15T07:59:28.714502+00:00",
   "updatedAt": "2025-04-27T08:18:13Z",
@@ -25,6 +25,6 @@
   "isReferenceServer": false,
   "isCommunityServer": true,
   "githubLatestCommit": "bf2c0261311c2fb4b2175e378080c8fa66c52289",
-  "githubForks": 718,
+  "githubForks": 723,
   "licenseType": "AGPL-3.0"
 }
diff --git a/server/src/data/split/05e19a5d-df7f-497e-9272-b36744db5cd5_cognee.json b/server/src/data/split/05e19a5d-df7f-497e-9272-b36744db5cd5_cognee.json
@@ -16,7 +16,7 @@
   ],
   "requiresApiKey": false,
   "isRecommended": false,
-  "githubStars": 2051,
+  "githubStars": 2056,
   "downloadCount": 0,
   "createdAt": "2025-03-17T08:29:22.149254+00:00",
   "updatedAt": "2025-04-25T22:03:05Z",

diff --git a/server/src/data/split/0730e308-7fb2-452e-a423-07c6c90e4ed0_mcpazuredevopsserver.json b/server/src/data/split/0730e308-7fb2-452e-a423-07c6c90e4ed0_mcpazuredevopsserver.json
@@ -16,7 +16,7 @@
   ],
   "requiresApiKey": false,
   "isRecommended": false,
-  "githubStars": 30,
+  "githubStars": 31,
   "downloadCount": 0,
   "createdAt": "2025-04-26T13:41:51.295Z",
   "updatedAt": "2025-04-09T19:07:12Z",

diff --git a/server/src/data/split/075a1d44-e621-461f-b3ae-b454ef1ef34b_deepseekreasoner.json b/server/src/data/split/075a1d44-e621-461f-b3ae-b454ef1ef34b_deepseekreasoner.json
@@ -16,7 +16,7 @@
   ],
   "requiresApiKey": false,
   "isRecommended": false,
-  "githubStars": 40,
+  "githubStars": 39,
   "downloadCount": 0,
   "createdAt": "2025-03-17T08:29:22.634233+00:00",
   "updatedAt": "2025-01-29T07:16:04Z",

diff --git a/server/src/data/split/078c5c1e-892f-46fd-8646-e9a884f36fc0_zig.json b/server/src/data/split/078c5c1e-892f-46fd-8646-e9a884f36fc0_zig.json
@@ -16,7 +16,7 @@
   ],
   "requiresApiKey": false,
   "isRecommended": false,
-  "githubStars": 12,
+  "githubStars": 11,
   "downloadCount": 0,
   "createdAt": "2025-03-17T08:29:24.291233+00:00",
   "updatedAt": "2025-01-09T07:29:20Z",

diff --git a/server/src/data/split/07eb49c7-9a55-4a00-b1a2-ba7fbef84fe6_devdb.json b/server/src/data/split/07eb49c7-9a55-4a00-b1a2-ba7fbef84fe6_devdb.json
@@ -9,7 +9,7 @@
   "tags": [],
   "requiresApiKey": false,
   "isRecommended": false,
-  "githubStars": 870,
+  "githubStars": 871,
   "downloadCount": 0,
   "createdAt": "2025-04-27T14:08:34.434Z",
   "updatedAt": "2025-04-24T17:55:44Z",

diff --git a/server/src/data/split/07fc60ed-12ed-41c5-90ea-9a4d0b1fa829_linear.json b/server/src/data/split/07fc60ed-12ed-41c5-90ea-9a4d0b1fa829_linear.json
@@ -16,7 +16,7 @@
   ],
   "requiresApiKey": false,
   "isRecommended": false,
-  "githubStars": 60,
+  "githubStars": 63,
   "downloadCount": 0,
   "createdAt": "2025-03-24T02:03:10.166753+00:00",
   "updatedAt": "2025-04-02T11:25:52Z",

diff --git a/server/src/data/split/08148a89-fbec-45bd-a137-12ff0b039a0e_notion.json b/server/src/data/split/08148a89-fbec-45bd-a137-12ff0b039a0e_notion.json
@@ -16,7 +16,7 @@
   ],
   "requiresApiKey": false,
   "isRecommended": false,
-  "githubStars": 80,
+  "githubStars": 81,
   "downloadCount": 0,
   "createdAt": "2025-03-17T08:29:22.634233+00:00",
   "updatedAt": "2025-02-01T02:13:16Z",

diff --git a/server/src/data/split/082d1c81-787d-4a55-abd4-8b6efbe26674_comfyui.json b/server/src/data/split/082d1c81-787d-4a55-abd4-8b6efbe26674_comfyui.json
@@ -16,7 +16,7 @@
   ],
   "requiresApiKey": false,
   "isRecommended": false,
-  "githubStars": 46,
+  "githubStars": 47,
   "downloadCount": 0,
   "createdAt": "2025-03-17T08:29:25.041553+00:00",
   "updatedAt": "2025-03-06T22:15:45Z",
@@ -25,6 +25,6 @@
   "isReferenceServer": false,
   "isCommunityServer": true,
   "githubLatestCommit": "26d5fbc7acb53394c0cd3037b0387ea096bf251e",
-  "githubForks": 11,
+  "githubForks": 12,
   "licenseType": "Apache-2.0"
 }
diff --git a/server/src/data/split/0948e941-3485-4192-9cff-cf951319ed38_evmmcpserver.json b/server/src/data/split/0948e941-3485-4192-9cff-cf951319ed38_evmmcpserver.json
@@ -9,7 +9,7 @@
   "tags": [],
   "requiresApiKey": false,
   "isRecommended": false,
-  "githubStars": 212,
+  "githubStars": 214,
   "downloadCount": 0,
   "createdAt": "2025-04-27T14:08:34.434Z",
   "updatedAt": "2025-03-22T20:17:47Z",

diff --git a/server/src/data/split/0961d473-0426-469f-b997-6f7df47445ac_curl.json b/server/src/data/split/0961d473-0426-469f-b997-6f7df47445ac_curl.json
@@ -16,7 +16,7 @@
   ],
   "requiresApiKey": false,
   "isRecommended": false,
-  "githubStars": 40,
+  "githubStars": 41,
   "downloadCount": 0,
   "createdAt": "2025-03-17T08:29:22.634233+00:00",
   "updatedAt": "2024-12-05T07:13:54Z",