feat: Robust GitHub API rate limit detection and graceful exit (#9087)

Author: tobiu

apps/devrank/services/GitHub.mjs

@@ -100,6 +100,32 @@ class GitHub extends Base {
         if (remaining !== null) this.rateLimit.remaining = parseInt(remaining, 10);
         if (reset !== null) this.rateLimit.reset = parseInt(reset, 10);
         if (limit !== null) this.rateLimit.limit = parseInt(limit, 10);
+
+        // Debug: Warn if headers are missing but we are at default (implying no update ever happened)
+        if (remaining === null && this.rateLimit.remaining === 5000) {
+            // Only log once or sparsely to avoid spam
+            if (!this._headerWarned) {
+                console.warn('[GitHub] Warning: `x-rate-limit-*` headers not found. Falling back to body (if available).');
+                this._headerWarned = true;
+            }
+        }
+    }
+
+    /**
+     * Updates rate limit from GraphQL body.
+     * @param {Object} rateLimit
+     * @private
+     */
+    #updateFromBody(rateLimit) {
+        if (!rateLimit) return;
+        
+        if (rateLimit.remaining !== undefined) this.rateLimit.remaining = rateLimit.remaining;
+        if (rateLimit.limit !== undefined)     this.rateLimit.limit     = rateLimit.limit;
+        
+        if (rateLimit.resetAt) {
+            // GraphQL returns ISO string, we store epoch seconds
+            this.rateLimit.reset = Math.floor(new Date(rateLimit.resetAt).getTime() / 1000);
+        }
     }
 
     /**
@@ -140,6 +166,11 @@ class GitHub extends Base {
 
             const json = await response.json();
 
+            // Hook for body-based rate limit (more reliable for GraphQL)
+            if (json.data?.rateLimit) {
+                this.#updateFromBody(json.data.rateLimit);
+            }
+
             if (json.errors) {
                 // Sometimes 502s come as 200 OK with errors body
                 const isGatewayError = json.errors.some(e => e.message?.includes('502') || e.message?.includes('504'));
@@ -197,6 +228,9 @@ class GitHub extends Base {
             this.#updateRateLimit(response.headers);
 
             if (!response.ok) {
+                if (response.status === 403) {
+                    this.rateLimit.remaining = 0;
+                }
                 throw new Error(`REST Error: ${response.status} ${response.statusText}`);
             }
 

apps/devrank/services/Spider.mjs

@@ -514,6 +514,12 @@ class Spider extends Base {
                 .filter(c => c.type === 'User')
                 .map(c => c.login);
         } catch (e) {
+            // Kill-switch for rate limits
+            if (e.message.includes('403') || e.message.includes('rate limit')) {
+                console.warn(`[Spider] 🚨 Rate limit hit scanning ${fullName}. Forcing shutdown.`);
+                GitHub.rateLimit.remaining = 0;
+            }
+
             console.error(`[Spider] Failed to fetch contributors for ${fullName}: ${e.message}`);
             return [];
         }

apps/devrank/services/Updater.mjs

@@ -87,6 +87,12 @@ class Updater extends Base {
                 }
             } catch (error) {
                 console.log(`[${login}] FAILED: ${error.message}`);
+                
+                // Kill-switch: If we hit a rate limit error, force internal state to 0 to trigger graceful shutdown
+                if (error.message.includes('rate limit')) {
+                    console.warn(`[Updater] 🚨 Rate limit hit for ${login}. Forcing shutdown sequence.`);
+                    GitHub.rateLimit.remaining = 0;
+                }
             }
         };
 
@@ -160,6 +166,7 @@ class Updater extends Base {
         // 1. Fetch Basic Profile & Social Accounts (GraphQL)
         const profileQuery = `
             query { 
+                rateLimit { remaining limit resetAt }
                 user(login: "${username}") { 
                     createdAt 
                     avatarUrl 
@@ -227,7 +234,9 @@ class Updater extends Base {
         const contribData = {};
 
         const fetchYears = async (fromYear, toYear) => {
-            let query = `query { user(login: "${username}") {`;
+            let query = `query { 
+                rateLimit { remaining limit resetAt }
+                user(login: "${username}") {`;
             for (let year = fromYear; year <= toYear; year++) {
                 query += ` y${year}: contributionsCollection(from: "${year}-01-01T00:00:00Z", to: "${year}-12-31T23:59:59Z") { 
                     totalCommitContributions