From 3352dabe8850dc388db35adcc753cdeb5714b221 Mon Sep 17 00:00:00 2001 From: Alison Hawk Date: Fri, 12 Sep 2025 06:00:44 -0600 Subject: [PATCH 1/7] Enhance Telegram and Twitter API integration: improve error handling, track API usage, and refactor message fetching logic --- src/fetchTelegramMessages.ts | 102 ++++++++++++++++++++++------------- src/index.ts | 21 +++----- src/twitterApi.ts | 55 +++++++++++++++---- src/utils/redisUtils.ts | 60 +++++++++++++++++++++ 4 files changed, 179 insertions(+), 59 deletions(-) diff --git a/src/fetchTelegramMessages.ts b/src/fetchTelegramMessages.ts index b2f945b..1f86a6c 100644 --- a/src/fetchTelegramMessages.ts +++ b/src/fetchTelegramMessages.ts @@ -1,48 +1,78 @@ -import { TelegramClient } from "telegram"; -import { Api } from "telegram"; +import { Api, TelegramClient } from "telegram"; export type TelegramMessage = { id: string; content: string; channelId: string }; export async function fetchTelegramMessages( - client: TelegramClient, - channel: string + client: TelegramClient, + channel: string ): Promise { - if (!channel) { - throw new Error("TG_CHANNEL environment variable is not set."); + if (!channel) { + throw new Error("TG_CHANNEL environment variable is not set."); + } + + // Fetch channel entity to get the actual channel ID + let entity: Api.Channel; + try { + const resolved = await client.getEntity(channel); + if (resolved instanceof Api.Channel) { + entity = resolved; + } else if (resolved instanceof Api.ChannelForbidden) { + throw new Error(`TG_CHANNEL "${channel}" is a private/forbidden channel; cannot fetch history.`); + } else { + throw new Error(`TG_CHANNEL "${channel}" is not a channel-type peer.`); + } + } catch (e) { + throw new Error(`Failed to resolve TG_CHANNEL "${channel}": ${e instanceof Error ? e.message : e}`); + } + + const channelId = String(entity.id); + + // Fetch the latest 10 messages + const messages = await client.invoke( + new Api.messages.GetHistory({ + peer: entity, + limit: 10, + }) + ); + + const out: TelegramMessage[] = []; + + if ("messages" in messages) { + for (const msg of messages.messages as any[]) { + const id = typeof msg?.id === 'number' || typeof msg?.id === 'string' ? String(msg.id) : null; + const content = typeof msg?.message === 'string' ? msg.message : ''; + if (!id || !content) continue; // skip service/media-only messages + const formatted = { id, content, channelId }; + out.push(formatted); + console.log(formatted); } - // Fetch channel entity to get the actual channel ID - let entity: Api.Channel; + } else { + console.log("No messages property found in response:", messages); + } + + // Track API usage after successful fetch + if (process.env.API_ID) { + let accountHandle: string; + try { - const resolved = await client.getEntity(channel); - if (resolved instanceof Api.Channel) { - entity = resolved; - } else if (resolved instanceof Api.ChannelForbidden) { - throw new Error(`TG_CHANNEL "${channel}" is a private/forbidden channel; cannot fetch history.`); + // Automatically get the logged-in Telegram account + const me = await client.getMe(); + if (me) { + if (me.username && me.username.length > 0) { + accountHandle = `@${me.username}`; // use username if available } else { - throw new Error(`TG_CHANNEL "${channel}" is not a channel-type peer.`); + accountHandle = String(me.id); // fallback to numeric Telegram ID } + } else { + accountHandle = String(channelId); // fallback if getMe returns nothing + } } catch (e) { - throw new Error(`Failed to resolve TG_CHANNEL \"${channel}\": ${e instanceof Error ? e.message : e}`); + accountHandle = String(channelId); // fallback on error } - const channelId = String(entity.id); - const messages = await client.invoke( - new Api.messages.GetHistory({ - peer: entity, - limit: 10, - }) - ); - const out: TelegramMessage[] = []; - if ("messages" in messages) { - for (const msg of messages.messages as any[]) { - const id = typeof msg?.id === 'number' || typeof msg?.id === 'string' ? String(msg.id) : null; - const content = typeof msg?.message === 'string' ? msg.message : ''; - if (!id || !content) continue; // skip service/media-only - const formatted = { id, content, channelId }; - out.push(formatted); - console.log(formatted); - } - } else { - console.log("No messages property found in response:", messages); - } - return out; + + const { trackApiKeyUsage } = await import('./utils/redisUtils'); + await trackApiKeyUsage(process.env.API_ID as string, accountHandle); + } + + return out; } diff --git a/src/index.ts b/src/index.ts index 18fff1b..88e2d8f 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,10 +1,9 @@ -import { TelegramClient } from "telegram"; -import { StringSession } from "telegram/sessions"; +import 'dotenv/config'; import input from "input"; // interactive input for login import cron from 'node-cron'; -import { runRedisOperation } from './utils/redisUtils'; +import { TelegramClient } from "telegram"; +import { StringSession } from "telegram/sessions"; import { fetchTelegramMessages } from './fetchTelegramMessages'; -import 'dotenv/config'; // Replace these with your values const apiId = Number(process.env.API_ID); @@ -42,6 +41,10 @@ async function startTelegramCron() { // Run once at startup try { await fetchTelegramMessages(client, process.env.TG_CHANNEL!); + // Print Telegram API usage stats once + const { getApiKeyUsage } = await import('./utils/redisUtils'); + const usage = await getApiKeyUsage(process.env.API_ID as string); + console.log('Telegram API usage:', usage); } catch (err) { console.error("Startup Telegram fetch failed:", err); } @@ -51,6 +54,7 @@ async function startTelegramCron() { console.log('Refetching Telegram messages...'); try { await fetchTelegramMessages(client, process.env.TG_CHANNEL!); + // No duplicate print of Telegram API usage } catch (err) { console.error('Scheduled Telegram fetch failed:', err); } @@ -64,14 +68,5 @@ startTelegramCron().catch((err) => { }); -async function main() { - await runRedisOperation(async (client) => { - await client.set('test-key', 'hello-redis'); - const value = await client.get('test-key'); - console.log('Read value from Redis:', value); - }); -} - -main(); diff --git a/src/twitterApi.ts b/src/twitterApi.ts index b2cf22e..67f64dc 100644 --- a/src/twitterApi.ts +++ b/src/twitterApi.ts @@ -1,8 +1,35 @@ -import cron from 'node-cron'; import dotenv from 'dotenv'; +import cron from 'node-cron'; dotenv.config(); -export async function fetchHomeTimeline(seenTweetIds: string[] = []): Promise> { +async function fetchViewerAccount(): Promise<{ screenName: string; userId: string } | null> { + const url = "https://x.com/i/api/graphql/jMaTSZ5dqXctUg5f97R6xw/Viewer"; + + const headers = { + "authorization": `Bearer ${process.env.BEARER}`, + "x-csrf-token": process.env.CSRF_TOKEN as string, + "cookie": `auth_token=${process.env.AUTH_TOKEN}; ct0=${process.env.CSRF_TOKEN}`, + }; + + const res = await fetch(url, { method: "GET", headers }); + if (!res.ok) { + console.error("Viewer API request failed:", res.status, res.statusText); + return null; + } + + const data = await res.json(); + const user = data?.data?.viewer?.user_results?.result; + if (!user) return null; + + return { + screenName: user.legacy?.screen_name, + userId: user.rest_id, + }; +} + +export async function fetchHomeTimeline( + seenTweetIds: string[] = [] +): Promise> { const queryId = "wEpbv0WrfwV6y2Wlf0fxBQ"; const url = `https://x.com/i/api/graphql/${queryId}/HomeTimeline`; @@ -93,13 +120,12 @@ export async function fetchHomeTimeline(seenTweetIds: string[] = []): Promise = []; + const tweets: Array<{ content: string; id: string; authorId: string }> = []; const seenTweetIdsSet = new Set(seenTweetIds); try { - // Twitter's actual GraphQL HomeTimeline response structure const instructions = timeline?.home?.home_timeline_urt?.instructions || []; for (const instruction of instructions) { @@ -124,6 +150,16 @@ export async function fetchHomeTimeline(seenTweetIds: string[] = []): Promise { console.log('Refetching Twitter timeline...'); try { const timeline = await fetchHomeTimeline(); - // Process the timeline data here (save to DB, send to another service, etc.) console.log('Fetched timeline:', timeline); } catch (err) { console.error('Scheduled Twitter timeline fetch failed:', err); } }); - diff --git a/src/utils/redisUtils.ts b/src/utils/redisUtils.ts index ba8f23c..e6e6dd2 100644 --- a/src/utils/redisUtils.ts +++ b/src/utils/redisUtils.ts @@ -1,5 +1,65 @@ import { createClient, RedisClientType } from 'redis'; +/** + * Increment API key usage stats in Redis. + * @param apiKey The API key to track + */ +export async function trackApiKeyUsage(apiKey: string, accountHandle?: string): Promise { + await runRedisOperation(async (client) => { + const key = `api_usage:${apiKey}`; + await client.hIncrBy(key, 'total_requests', 1); + const now = new Date().toLocaleString(); + await client.hSet(key, 'last_request', now); + if (accountHandle) { + await client.hSet(key, 'account_handle', accountHandle); + } + }); +} + +/** + * Get API key usage stats from Redis. + * @param apiKey The API key to query + * @returns Object with total_requests and last_request + */ +export async function getApiKeyUsage(apiKey: string): Promise<{ total_requests: number; last_request: string | null; account_handle?: string }> { + let result: { total_requests: number; last_request: string | null; account_handle?: string } = { total_requests: 0, last_request: null }; + await runRedisOperation(async (client) => { + const key = `api_usage:${apiKey}`; + const data = await client.hGetAll(key); + result.total_requests = data.total_requests ? parseInt(data.total_requests) : 0; + result.last_request = data.last_request ? data.last_request : null; + if (data.account_handle) { + result.account_handle = data.account_handle; + } + }); + return result; +} + + + +// Example: Use environment variables for API keys +async function main() { + const telegramUsage = await getApiKeyUsage(process.env.API_ID as string); + console.log('Telegram API usage:', { + total_requests: telegramUsage.total_requests, + last_request: telegramUsage.last_request || 'No last Telegram request recorded.', + account_handle: telegramUsage.account_handle || 'No account handle recorded.' + }); + + const twitterUsage = await getApiKeyUsage(process.env.AUTH_TOKEN as string); + console.log('Twitter API usage:', { + total_requests: twitterUsage.total_requests, + last_request: twitterUsage.last_request || 'No last Twitter request recorded.', + account_handle: twitterUsage.account_handle || 'No account handle recorded.' + }); +} + +// Only run main if this file is executed directly (not imported) +if (require.main === module) { + main(); +} + + export async function runRedisOperation(operation: (client: RedisClientType) => Promise): Promise { const client: RedisClientType = createClient({ url: 'redis://localhost:6379' From d82070aa4d8cfa573b99bbc62cda9d9c20af7140 Mon Sep 17 00:00:00 2001 From: Alison Hawk Date: Fri, 12 Sep 2025 06:09:36 -0600 Subject: [PATCH 2/7] Refactor API usage logging: standardize output format for Telegram and Twitter API usage stats --- src/index.ts | 6 +++++- src/twitterApi.ts | 6 +++++- src/utils/redisUtils.ts | 6 +++--- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/index.ts b/src/index.ts index 88e2d8f..ff6792a 100644 --- a/src/index.ts +++ b/src/index.ts @@ -44,7 +44,11 @@ async function startTelegramCron() { // Print Telegram API usage stats once const { getApiKeyUsage } = await import('./utils/redisUtils'); const usage = await getApiKeyUsage(process.env.API_ID as string); - console.log('Telegram API usage:', usage); + console.log('Telegram API usage:', { + total_requests: usage.total_requests, + last_request: usage.last_request, + account_id: usage.account_handle + }); } catch (err) { console.error("Startup Telegram fetch failed:", err); } diff --git a/src/twitterApi.ts b/src/twitterApi.ts index 67f64dc..d076d33 100644 --- a/src/twitterApi.ts +++ b/src/twitterApi.ts @@ -173,7 +173,11 @@ async function main() { const { getApiKeyUsage } = await import('./utils/redisUtils'); const usage = await getApiKeyUsage(process.env.AUTH_TOKEN as string); - console.log('Twitter API usage:', usage); + console.log('Twitter API usage:', { + total_requests: usage.total_requests, + last_request: usage.last_request, + account_id: usage.account_handle + }); } catch (err) { console.error('fetchHomeTimeline failed:', err instanceof Error ? err.message : err); process.exit(1); diff --git a/src/utils/redisUtils.ts b/src/utils/redisUtils.ts index e6e6dd2..faba164 100644 --- a/src/utils/redisUtils.ts +++ b/src/utils/redisUtils.ts @@ -8,7 +8,7 @@ export async function trackApiKeyUsage(apiKey: string, accountHandle?: string): await runRedisOperation(async (client) => { const key = `api_usage:${apiKey}`; await client.hIncrBy(key, 'total_requests', 1); - const now = new Date().toLocaleString(); + const now = new Date().toISOString(); await client.hSet(key, 'last_request', now); if (accountHandle) { await client.hSet(key, 'account_handle', accountHandle); @@ -43,14 +43,14 @@ async function main() { console.log('Telegram API usage:', { total_requests: telegramUsage.total_requests, last_request: telegramUsage.last_request || 'No last Telegram request recorded.', - account_handle: telegramUsage.account_handle || 'No account handle recorded.' + account_id: telegramUsage.account_handle || 'No account handle recorded.' }); const twitterUsage = await getApiKeyUsage(process.env.AUTH_TOKEN as string); console.log('Twitter API usage:', { total_requests: twitterUsage.total_requests, last_request: twitterUsage.last_request || 'No last Twitter request recorded.', - account_handle: twitterUsage.account_handle || 'No account handle recorded.' + account_id: twitterUsage.account_handle || 'No account id recorded.' }); } From 686d5bfed2faecc65b2667ea8a07a630903e7cd6 Mon Sep 17 00:00:00 2001 From: Alison Hawk Date: Fri, 12 Sep 2025 06:23:31 -0600 Subject: [PATCH 3/7] Refactor Redis API key usage tracking: implement hashing for apiKey and optimize Redis operations --- src/utils/redisUtils.ts | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/src/utils/redisUtils.ts b/src/utils/redisUtils.ts index faba164..b3b7d25 100644 --- a/src/utils/redisUtils.ts +++ b/src/utils/redisUtils.ts @@ -1,3 +1,4 @@ +import crypto from 'crypto'; import { createClient, RedisClientType } from 'redis'; /** @@ -6,13 +7,22 @@ import { createClient, RedisClientType } from 'redis'; */ export async function trackApiKeyUsage(apiKey: string, accountHandle?: string): Promise { await runRedisOperation(async (client) => { - const key = `api_usage:${apiKey}`; - await client.hIncrBy(key, 'total_requests', 1); - const now = new Date().toISOString(); - await client.hSet(key, 'last_request', now); - if (accountHandle) { - await client.hSet(key, 'account_handle', accountHandle); + if (!apiKey?.trim()) { + console.warn('trackApiKeyUsage: empty apiKey; skipping'); + return; } + // Hash the apiKey to avoid storing raw secrets in Redis + const hash = crypto.createHash('sha256').update(apiKey).digest('hex').slice(0, 32); + const key = `api_usage:${hash}`; + const now = new Date().toISOString(); + await client + .multi() + .hIncrBy(key, 'total_requests', 1) + .hSet(key, { + last_request: now, + ...(accountHandle ? { account_handle: accountHandle } : {}), + }) + .exec(); }); } @@ -24,7 +34,9 @@ export async function trackApiKeyUsage(apiKey: string, accountHandle?: string): export async function getApiKeyUsage(apiKey: string): Promise<{ total_requests: number; last_request: string | null; account_handle?: string }> { let result: { total_requests: number; last_request: string | null; account_handle?: string } = { total_requests: 0, last_request: null }; await runRedisOperation(async (client) => { - const key = `api_usage:${apiKey}`; + // Hash the apiKey to avoid storing raw secrets in Redis + const hash = crypto.createHash('sha256').update(apiKey).digest('hex').slice(0, 32); + const key = `api_usage:${hash}`; const data = await client.hGetAll(key); result.total_requests = data.total_requests ? parseInt(data.total_requests) : 0; result.last_request = data.last_request ? data.last_request : null; From 897070f84fae7146d3c6a2e4925d80a5835c21aa Mon Sep 17 00:00:00 2001 From: Alison Hawk Date: Fri, 12 Sep 2025 06:59:34 -0600 Subject: [PATCH 4/7] Enhance Redis API key usage tracking: add error handling and improve input validation --- src/utils/redisUtils.ts | 65 ++++++++++++++++++++++++++--------------- 1 file changed, 41 insertions(+), 24 deletions(-) diff --git a/src/utils/redisUtils.ts b/src/utils/redisUtils.ts index b3b7d25..9987a34 100644 --- a/src/utils/redisUtils.ts +++ b/src/utils/redisUtils.ts @@ -11,18 +11,22 @@ export async function trackApiKeyUsage(apiKey: string, accountHandle?: string): console.warn('trackApiKeyUsage: empty apiKey; skipping'); return; } - // Hash the apiKey to avoid storing raw secrets in Redis - const hash = crypto.createHash('sha256').update(apiKey).digest('hex').slice(0, 32); - const key = `api_usage:${hash}`; - const now = new Date().toISOString(); - await client - .multi() - .hIncrBy(key, 'total_requests', 1) - .hSet(key, { - last_request: now, - ...(accountHandle ? { account_handle: accountHandle } : {}), - }) - .exec(); + try { + // Hash the apiKey to avoid storing raw secrets in Redis + const hash = crypto.createHash('sha256').update(apiKey).digest('hex').slice(0, 32); + const key = `api_usage:${hash}`; + const now = new Date().toISOString(); + await client + .multi() + .hIncrBy(key, 'total_requests', 1) + .hSet(key, { + last_request: now, + ...(accountHandle ? { account_handle: accountHandle } : {}), + }) + .exec(); + } catch (err) { + console.warn('trackApiKeyUsage: non-fatal Redis error; proceeding without usage update', err); + } }); } @@ -33,6 +37,9 @@ export async function trackApiKeyUsage(apiKey: string, accountHandle?: string): */ export async function getApiKeyUsage(apiKey: string): Promise<{ total_requests: number; last_request: string | null; account_handle?: string }> { let result: { total_requests: number; last_request: string | null; account_handle?: string } = { total_requests: 0, last_request: null }; + if (!apiKey?.trim()) { + return result; + } await runRedisOperation(async (client) => { // Hash the apiKey to avoid storing raw secrets in Redis const hash = crypto.createHash('sha256').update(apiKey).digest('hex').slice(0, 32); @@ -51,19 +58,29 @@ export async function getApiKeyUsage(apiKey: string): Promise<{ total_requests: // Example: Use environment variables for API keys async function main() { - const telegramUsage = await getApiKeyUsage(process.env.API_ID as string); - console.log('Telegram API usage:', { - total_requests: telegramUsage.total_requests, - last_request: telegramUsage.last_request || 'No last Telegram request recorded.', - account_id: telegramUsage.account_handle || 'No account handle recorded.' - }); + const apiId = process.env.API_ID; + if (apiId?.trim()) { + const telegramUsage = await getApiKeyUsage(apiId); + console.log('Telegram API usage:', { + total_requests: telegramUsage.total_requests, + last_request: telegramUsage.last_request || 'No last Telegram request recorded.', + account_id: telegramUsage.account_handle || 'No account id recorded.' + }); + } else { + console.log('Telegram API usage: API_ID not set.'); + } - const twitterUsage = await getApiKeyUsage(process.env.AUTH_TOKEN as string); - console.log('Twitter API usage:', { - total_requests: twitterUsage.total_requests, - last_request: twitterUsage.last_request || 'No last Twitter request recorded.', - account_id: twitterUsage.account_handle || 'No account id recorded.' - }); + const authToken = process.env.AUTH_TOKEN; + if (authToken?.trim()) { + const twitterUsage = await getApiKeyUsage(authToken); + console.log('Twitter API usage:', { + total_requests: twitterUsage.total_requests, + last_request: twitterUsage.last_request || 'No last Twitter request recorded.', + account_id: twitterUsage.account_handle || 'No account id recorded.' + }); + } else { + console.log('Twitter API usage: AUTH_TOKEN not set.'); + } } // Only run main if this file is executed directly (not imported) From ea34ff994dfca8b12aa2d3ec60c3eca0fbddff00 Mon Sep 17 00:00:00 2001 From: Alison Hawk Date: Fri, 12 Sep 2025 07:34:51 -0600 Subject: [PATCH 5/7] Enhance Telegram and Twitter API usage tracking: refactor account ID handling and improve Redis key management --- src/fetchTelegramMessages.ts | 30 +++----- src/index.ts | 2 +- src/twitterApi.ts | 13 ++-- src/utils/redisUtils.ts | 141 ++++++++++++++++++----------------- 4 files changed, 90 insertions(+), 96 deletions(-) diff --git a/src/fetchTelegramMessages.ts b/src/fetchTelegramMessages.ts index 1f86a6c..7bc515f 100644 --- a/src/fetchTelegramMessages.ts +++ b/src/fetchTelegramMessages.ts @@ -1,14 +1,16 @@ import { Api, TelegramClient } from "telegram"; +import { trackApiKeyUsage } from './utils/redisUtils'; -export type TelegramMessage = { id: string; content: string; channelId: string }; +export type TelegramMessages = { id: string; content: string; channelId: string }; export async function fetchTelegramMessages( client: TelegramClient, channel: string -): Promise { +): Promise { if (!channel) { throw new Error("TG_CHANNEL environment variable is not set."); } + const apiId = process.env.API_ID; // Fetch channel entity to get the actual channel ID let entity: Api.Channel; @@ -35,7 +37,7 @@ export async function fetchTelegramMessages( }) ); - const out: TelegramMessage[] = []; + const out: TelegramMessages[] = []; if ("messages" in messages) { for (const msg of messages.messages as any[]) { @@ -51,27 +53,19 @@ export async function fetchTelegramMessages( } // Track API usage after successful fetch - if (process.env.API_ID) { - let accountHandle: string; - + if (apiId) { + let accountId: string; try { - // Automatically get the logged-in Telegram account const me = await client.getMe(); - if (me) { - if (me.username && me.username.length > 0) { - accountHandle = `@${me.username}`; // use username if available - } else { - accountHandle = String(me.id); // fallback to numeric Telegram ID - } + if (me && me.id) { + accountId = String(me.id); } else { - accountHandle = String(channelId); // fallback if getMe returns nothing + throw new Error('Unable to determine Telegram account ID'); } } catch (e) { - accountHandle = String(channelId); // fallback on error + throw new Error('Unable to determine Telegram account ID'); } - - const { trackApiKeyUsage } = await import('./utils/redisUtils'); - await trackApiKeyUsage(process.env.API_ID as string, accountHandle); + await trackApiKeyUsage(apiId, accountId); } return out; diff --git a/src/index.ts b/src/index.ts index ff6792a..172eab0 100644 --- a/src/index.ts +++ b/src/index.ts @@ -47,7 +47,7 @@ async function startTelegramCron() { console.log('Telegram API usage:', { total_requests: usage.total_requests, last_request: usage.last_request, - account_id: usage.account_handle + account_id: usage.account_id }); } catch (err) { console.error("Startup Telegram fetch failed:", err); diff --git a/src/twitterApi.ts b/src/twitterApi.ts index d076d33..e22259a 100644 --- a/src/twitterApi.ts +++ b/src/twitterApi.ts @@ -1,5 +1,8 @@ import dotenv from 'dotenv'; import cron from 'node-cron'; +import { getApiKeyUsage } from './utils/redisUtils'; +import { trackApiKeyUsage } from './utils/redisUtils'; +const AUTH_TOKEN = process.env.AUTH_TOKEN; dotenv.config(); async function fetchViewerAccount(): Promise<{ screenName: string; userId: string } | null> { @@ -151,13 +154,12 @@ export async function fetchHomeTimeline( } // Track API usage after successful fetch - if (process.env.AUTH_TOKEN) { + if (AUTH_TOKEN) { // Use TWITTER_ID from environment variable for account handle (no '@' prefix) const accountHandle = process.env.TWITTER_ID ? process.env.TWITTER_ID : "unknown"; console.log("Authenticated account:", accountHandle); - const { trackApiKeyUsage } = await import('./utils/redisUtils'); - await trackApiKeyUsage(process.env.AUTH_TOKEN as string, accountHandle); + await trackApiKeyUsage(AUTH_TOKEN, accountHandle); } return tweets; @@ -169,14 +171,11 @@ async function main() { try { const data = await fetchHomeTimeline(); - console.log(JSON.stringify(data, null, 2)); - - const { getApiKeyUsage } = await import('./utils/redisUtils'); const usage = await getApiKeyUsage(process.env.AUTH_TOKEN as string); console.log('Twitter API usage:', { total_requests: usage.total_requests, last_request: usage.last_request, - account_id: usage.account_handle + account_id: usage.account_id }); } catch (err) { console.error('fetchHomeTimeline failed:', err instanceof Error ? err.message : err); diff --git a/src/utils/redisUtils.ts b/src/utils/redisUtils.ts index 9987a34..cce7d62 100644 --- a/src/utils/redisUtils.ts +++ b/src/utils/redisUtils.ts @@ -1,33 +1,48 @@ import crypto from 'crypto'; -import { createClient, RedisClientType } from 'redis'; +import { createClient } from 'redis'; -/** - * Increment API key usage stats in Redis. - * @param apiKey The API key to track - */ -export async function trackApiKeyUsage(apiKey: string, accountHandle?: string): Promise { - await runRedisOperation(async (client) => { - if (!apiKey?.trim()) { - console.warn('trackApiKeyUsage: empty apiKey; skipping'); - return; - } - try { - // Hash the apiKey to avoid storing raw secrets in Redis - const hash = crypto.createHash('sha256').update(apiKey).digest('hex').slice(0, 32); - const key = `api_usage:${hash}`; - const now = new Date().toISOString(); - await client - .multi() - .hIncrBy(key, 'total_requests', 1) - .hSet(key, { - last_request: now, - ...(accountHandle ? { account_handle: accountHandle } : {}), - }) - .exec(); - } catch (err) { - console.warn('trackApiKeyUsage: non-fatal Redis error; proceeding without usage update', err); +// Singleton Redis client +const redisClient = createClient({ url: 'redis://localhost:6379' }); +let redisConnected = false; + +redisClient.on('error', (err) => { + console.error('Redis Client Error', err); +}); + +async function ensureRedisConnected() { + if (!redisConnected) { + await redisClient.connect(); + redisConnected = true; + } +} +export async function trackApiKeyUsage(apiKey: string, accountId?: string): Promise { + if (!apiKey?.trim()) { + console.warn('trackApiKeyUsage: empty apiKey; skipping'); + return; + } + + try { + await ensureRedisConnected(); + let key: string; + if (process.env.TWITTER_ACCOUNT_ID && apiKey === process.env.TWITTER_ACCOUNT_ID) { + key = `twitter_accounts:${apiKey}`; + } else if (process.env.TELEGRAM_ACCOUNT_ID && apiKey === process.env.TELEGRAM_ACCOUNT_ID) { + key = `telegram_accounts:${apiKey}`; + } else { + key = `api_usage:${apiKey}`; } - }); + const now = new Date().toISOString(); + await redisClient + .multi() + .hIncrBy(key, 'total_requests', 1) + .hSet(key, { + last_request: now, + ...(accountId ? { account_id: accountId } : {}), + }) + .exec(); + } catch (err) { + console.warn('trackApiKeyUsage: non-fatal Redis error; proceeding without usage update', err); + } } /** @@ -35,22 +50,31 @@ export async function trackApiKeyUsage(apiKey: string, accountHandle?: string): * @param apiKey The API key to query * @returns Object with total_requests and last_request */ -export async function getApiKeyUsage(apiKey: string): Promise<{ total_requests: number; last_request: string | null; account_handle?: string }> { - let result: { total_requests: number; last_request: string | null; account_handle?: string } = { total_requests: 0, last_request: null }; +export async function getApiKeyUsage(apiKey: string): Promise<{ total_requests: number; last_request: string | null; account_id?: string }> { + let result: { total_requests: number; last_request: string | null; account_id?: string } = { total_requests: 0, last_request: null }; if (!apiKey?.trim()) { return result; } - await runRedisOperation(async (client) => { - // Hash the apiKey to avoid storing raw secrets in Redis - const hash = crypto.createHash('sha256').update(apiKey).digest('hex').slice(0, 32); - const key = `api_usage:${hash}`; - const data = await client.hGetAll(key); + + try { + await ensureRedisConnected(); + let key: string; + if (process.env.TWITTER_ACCOUNT_ID && apiKey === process.env.TWITTER_ACCOUNT_ID) { + key = `twitter_accounts:${apiKey}`; + } else if (process.env.TELEGRAM_ACCOUNT_ID && apiKey === process.env.TELEGRAM_ACCOUNT_ID) { + key = `telegram_accounts:${apiKey}`; + } else { + key = `api_usage:${apiKey}`; + } + const data = await redisClient.hGetAll(key); result.total_requests = data.total_requests ? parseInt(data.total_requests) : 0; result.last_request = data.last_request ? data.last_request : null; - if (data.account_handle) { - result.account_handle = data.account_handle; + if (data.account_id) { + result.account_id = data.account_id; } - }); + } catch (err) { + console.error('Redis operation failed:', err); + } return result; } @@ -58,28 +82,28 @@ export async function getApiKeyUsage(apiKey: string): Promise<{ total_requests: // Example: Use environment variables for API keys async function main() { - const apiId = process.env.API_ID; - if (apiId?.trim()) { - const telegramUsage = await getApiKeyUsage(apiId); + const telegramAccountId = process.env.TELEGRAM_ACCOUNT_ID; + if (telegramAccountId?.trim()) { + const telegramUsage = await getApiKeyUsage(telegramAccountId); console.log('Telegram API usage:', { total_requests: telegramUsage.total_requests, last_request: telegramUsage.last_request || 'No last Telegram request recorded.', - account_id: telegramUsage.account_handle || 'No account id recorded.' + account_id: telegramUsage.account_id || 'No account id recorded.' }); } else { - console.log('Telegram API usage: API_ID not set.'); + console.log('Telegram API usage: TELEGRAM_ACCOUNT_ID not set.'); } - const authToken = process.env.AUTH_TOKEN; - if (authToken?.trim()) { - const twitterUsage = await getApiKeyUsage(authToken); + const twitterAccountId = process.env.TWITTER_ACCOUNT_ID; + if (twitterAccountId?.trim()) { + const twitterUsage = await getApiKeyUsage(twitterAccountId); console.log('Twitter API usage:', { total_requests: twitterUsage.total_requests, last_request: twitterUsage.last_request || 'No last Twitter request recorded.', - account_id: twitterUsage.account_handle || 'No account id recorded.' + account_id: twitterUsage.account_id || 'No account id recorded.' }); } else { - console.log('Twitter API usage: AUTH_TOKEN not set.'); + console.log('Twitter API usage: TWITTER_ACCOUNT_ID not set.'); } } @@ -89,26 +113,3 @@ if (require.main === module) { } -export async function runRedisOperation(operation: (client: RedisClientType) => Promise): Promise { - const client: RedisClientType = createClient({ - url: 'redis://localhost:6379' - }); - - client.on('error', (err) => { - console.error('Redis Client Error', err); - }); - - try { - await client.connect(); - await operation(client); - } catch (err) { - console.error('Redis operation failed:', err); - throw err; - } finally { - try { - await client.quit(); - } catch (quitErr) { - console.error('Error during Redis client quit:', quitErr); - } - } -} From 364040ef1b8cadc4cce39e7119ce1f8983c8bc73 Mon Sep 17 00:00:00 2001 From: Alison Hawk Date: Fri, 12 Sep 2025 11:05:47 -0600 Subject: [PATCH 6/7] Refactor API key usage tracking: update trackApiKeyUsage and getApiKeyUsage functions to accept accountId and platform parameters for improved clarity and consistency --- src/fetchTelegramMessages.ts | 2 +- src/index.ts | 6 ++- src/twitterApi.ts | 15 ++++---- src/utils/redisUtils.ts | 75 +++++++++++------------------------- 4 files changed, 35 insertions(+), 63 deletions(-) diff --git a/src/fetchTelegramMessages.ts b/src/fetchTelegramMessages.ts index 7bc515f..df72084 100644 --- a/src/fetchTelegramMessages.ts +++ b/src/fetchTelegramMessages.ts @@ -65,7 +65,7 @@ export async function fetchTelegramMessages( } catch (e) { throw new Error('Unable to determine Telegram account ID'); } - await trackApiKeyUsage(apiId, accountId); + await trackApiKeyUsage({ accountId, platform: 'telegram' }); } return out; diff --git a/src/index.ts b/src/index.ts index 172eab0..914444e 100644 --- a/src/index.ts +++ b/src/index.ts @@ -41,9 +41,11 @@ async function startTelegramCron() { // Run once at startup try { await fetchTelegramMessages(client, process.env.TG_CHANNEL!); - // Print Telegram API usage stats once + // Print Telegram API usage by accountId (not API_ID) const { getApiKeyUsage } = await import('./utils/redisUtils'); - const usage = await getApiKeyUsage(process.env.API_ID as string); + const me = await client.getMe(); + const accountId = String(me.id); + const usage = await getApiKeyUsage(accountId, 'telegram'); console.log('Telegram API usage:', { total_requests: usage.total_requests, last_request: usage.last_request, diff --git a/src/twitterApi.ts b/src/twitterApi.ts index e22259a..66c7d7e 100644 --- a/src/twitterApi.ts +++ b/src/twitterApi.ts @@ -154,12 +154,10 @@ export async function fetchHomeTimeline( } // Track API usage after successful fetch - if (AUTH_TOKEN) { - // Use TWITTER_ID from environment variable for account handle (no '@' prefix) - const accountHandle = process.env.TWITTER_ID ? process.env.TWITTER_ID : "unknown"; - console.log("Authenticated account:", accountHandle); - - await trackApiKeyUsage(AUTH_TOKEN, accountHandle); + if (process.env.TWITTER_ID) { + const accountId = process.env.TWITTER_ID; + console.log("Authenticated account:", accountId); + await trackApiKeyUsage({ accountId, platform: 'twitter' }); } return tweets; @@ -171,7 +169,10 @@ async function main() { try { const data = await fetchHomeTimeline(); - const usage = await getApiKeyUsage(process.env.AUTH_TOKEN as string); + const viewer = await fetchViewerAccount(); + const accountId = viewer?.userId ?? process.env.TWITTER_ACCOUNT_ID; + if (!accountId) throw new Error('Missing TWITTER_ACCOUNT_ID and Viewer lookup failed.'); + const usage = await getApiKeyUsage(accountId, 'twitter'); console.log('Twitter API usage:', { total_requests: usage.total_requests, last_request: usage.last_request, diff --git a/src/utils/redisUtils.ts b/src/utils/redisUtils.ts index cce7d62..89d7d36 100644 --- a/src/utils/redisUtils.ts +++ b/src/utils/redisUtils.ts @@ -15,21 +15,21 @@ async function ensureRedisConnected() { redisConnected = true; } } -export async function trackApiKeyUsage(apiKey: string, accountId?: string): Promise { - if (!apiKey?.trim()) { - console.warn('trackApiKeyUsage: empty apiKey; skipping'); +export async function trackApiKeyUsage({ accountId, platform }: { accountId: string, platform: 'telegram' | 'twitter' }): Promise { + if (!accountId?.trim()) { + console.warn('trackApiKeyUsage: empty accountId; skipping'); return; } try { await ensureRedisConnected(); let key: string; - if (process.env.TWITTER_ACCOUNT_ID && apiKey === process.env.TWITTER_ACCOUNT_ID) { - key = `twitter_accounts:${apiKey}`; - } else if (process.env.TELEGRAM_ACCOUNT_ID && apiKey === process.env.TELEGRAM_ACCOUNT_ID) { - key = `telegram_accounts:${apiKey}`; + if (platform === 'twitter') { + key = `twitter_accounts:${accountId}`; + } else if (platform === 'telegram') { + key = `telegram_accounts:${accountId}`; } else { - key = `api_usage:${apiKey}`; + key = `api_usage:${accountId}`; } const now = new Date().toISOString(); await redisClient @@ -37,7 +37,7 @@ export async function trackApiKeyUsage(apiKey: string, accountId?: string): Prom .hIncrBy(key, 'total_requests', 1) .hSet(key, { last_request: now, - ...(accountId ? { account_id: accountId } : {}), + account_id: accountId, }) .exec(); } catch (err) { @@ -46,25 +46,28 @@ export async function trackApiKeyUsage(apiKey: string, accountId?: string): Prom } /** - * Get API key usage stats from Redis. - * @param apiKey The API key to query + * Get API usage stats from Redis. + * @param accountId The account ID to query + * @param platform The platform ('telegram' or 'twitter') * @returns Object with total_requests and last_request */ -export async function getApiKeyUsage(apiKey: string): Promise<{ total_requests: number; last_request: string | null; account_id?: string }> { +export async function getApiKeyUsage(accountId: string, platform: 'telegram' | 'twitter'): Promise<{ total_requests: number; last_request: string | null; account_id?: string }> { let result: { total_requests: number; last_request: string | null; account_id?: string } = { total_requests: 0, last_request: null }; - if (!apiKey?.trim()) { + if (!accountId?.trim()) { return result; } - + if (platform !== 'twitter' && platform !== 'telegram') { + throw new Error('getApiKeyUsage: platform must be "twitter" or "telegram"'); + } try { await ensureRedisConnected(); let key: string; - if (process.env.TWITTER_ACCOUNT_ID && apiKey === process.env.TWITTER_ACCOUNT_ID) { - key = `twitter_accounts:${apiKey}`; - } else if (process.env.TELEGRAM_ACCOUNT_ID && apiKey === process.env.TELEGRAM_ACCOUNT_ID) { - key = `telegram_accounts:${apiKey}`; + if (platform === 'twitter') { + key = `twitter_accounts:${accountId}`; + } else if (platform === 'telegram') { + key = `telegram_accounts:${accountId}`; } else { - key = `api_usage:${apiKey}`; + throw new Error('getApiKeyUsage: platform must be "twitter" or "telegram"'); } const data = await redisClient.hGetAll(key); result.total_requests = data.total_requests ? parseInt(data.total_requests) : 0; @@ -79,37 +82,3 @@ export async function getApiKeyUsage(apiKey: string): Promise<{ total_requests: } - -// Example: Use environment variables for API keys -async function main() { - const telegramAccountId = process.env.TELEGRAM_ACCOUNT_ID; - if (telegramAccountId?.trim()) { - const telegramUsage = await getApiKeyUsage(telegramAccountId); - console.log('Telegram API usage:', { - total_requests: telegramUsage.total_requests, - last_request: telegramUsage.last_request || 'No last Telegram request recorded.', - account_id: telegramUsage.account_id || 'No account id recorded.' - }); - } else { - console.log('Telegram API usage: TELEGRAM_ACCOUNT_ID not set.'); - } - - const twitterAccountId = process.env.TWITTER_ACCOUNT_ID; - if (twitterAccountId?.trim()) { - const twitterUsage = await getApiKeyUsage(twitterAccountId); - console.log('Twitter API usage:', { - total_requests: twitterUsage.total_requests, - last_request: twitterUsage.last_request || 'No last Twitter request recorded.', - account_id: twitterUsage.account_id || 'No account id recorded.' - }); - } else { - console.log('Twitter API usage: TWITTER_ACCOUNT_ID not set.'); - } -} - -// Only run main if this file is executed directly (not imported) -if (require.main === module) { - main(); -} - - From 9e3cd4e19b9fca5c6315019f8d4400f4b1658577 Mon Sep 17 00:00:00 2001 From: Alison Hawk Date: Sat, 13 Sep 2025 12:27:42 -0600 Subject: [PATCH 7/7] Refactor API key usage retrieval: update getApiKeyUsage and trackApiKeyUsage functions to accept an object with accountId and platform parameters for improved clarity and error handling --- src/index.ts | 4 ++-- src/twitterApi.ts | 2 +- src/utils/redisUtils.ts | 16 +++++++++++----- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/src/index.ts b/src/index.ts index 914444e..0281120 100644 --- a/src/index.ts +++ b/src/index.ts @@ -4,6 +4,7 @@ import cron from 'node-cron'; import { TelegramClient } from "telegram"; import { StringSession } from "telegram/sessions"; import { fetchTelegramMessages } from './fetchTelegramMessages'; +import { getApiKeyUsage } from './utils/redisUtils'; // Replace these with your values const apiId = Number(process.env.API_ID); @@ -42,10 +43,9 @@ async function startTelegramCron() { try { await fetchTelegramMessages(client, process.env.TG_CHANNEL!); // Print Telegram API usage by accountId (not API_ID) - const { getApiKeyUsage } = await import('./utils/redisUtils'); const me = await client.getMe(); const accountId = String(me.id); - const usage = await getApiKeyUsage(accountId, 'telegram'); + const usage = await getApiKeyUsage({accountId, platform:'telegram'}); console.log('Telegram API usage:', { total_requests: usage.total_requests, last_request: usage.last_request, diff --git a/src/twitterApi.ts b/src/twitterApi.ts index 66c7d7e..eb65be7 100644 --- a/src/twitterApi.ts +++ b/src/twitterApi.ts @@ -172,7 +172,7 @@ async function main() { const viewer = await fetchViewerAccount(); const accountId = viewer?.userId ?? process.env.TWITTER_ACCOUNT_ID; if (!accountId) throw new Error('Missing TWITTER_ACCOUNT_ID and Viewer lookup failed.'); - const usage = await getApiKeyUsage(accountId, 'twitter'); + const usage = await getApiKeyUsage({accountId, platform:'twitter'}); console.log('Twitter API usage:', { total_requests: usage.total_requests, last_request: usage.last_request, diff --git a/src/utils/redisUtils.ts b/src/utils/redisUtils.ts index 89d7d36..fd38d19 100644 --- a/src/utils/redisUtils.ts +++ b/src/utils/redisUtils.ts @@ -29,7 +29,7 @@ export async function trackApiKeyUsage({ accountId, platform }: { accountId: str } else if (platform === 'telegram') { key = `telegram_accounts:${accountId}`; } else { - key = `api_usage:${accountId}`; + throw new Error('trackApiKeyUsage: platform must be "twitter" or "telegram"'); } const now = new Date().toISOString(); await redisClient @@ -51,7 +51,14 @@ export async function trackApiKeyUsage({ accountId, platform }: { accountId: str * @param platform The platform ('telegram' or 'twitter') * @returns Object with total_requests and last_request */ -export async function getApiKeyUsage(accountId: string, platform: 'telegram' | 'twitter'): Promise<{ total_requests: number; last_request: string | null; account_id?: string }> { + +interface dataType { + accountId: string, + platform: 'telegram' | 'twitter' +} + +export async function getApiKeyUsage(data: dataType): Promise<{ total_requests: number; last_request: string | null; account_id?: string }> { + const { accountId, platform } = data; let result: { total_requests: number; last_request: string | null; account_id?: string } = { total_requests: 0, last_request: null }; if (!accountId?.trim()) { return result; @@ -64,11 +71,10 @@ export async function getApiKeyUsage(accountId: string, platform: 'telegram' | ' let key: string; if (platform === 'twitter') { key = `twitter_accounts:${accountId}`; - } else if (platform === 'telegram') { - key = `telegram_accounts:${accountId}`; } else { - throw new Error('getApiKeyUsage: platform must be "twitter" or "telegram"'); + key = `telegram_accounts:${accountId}`; } + const data = await redisClient.hGetAll(key); result.total_requests = data.total_requests ? parseInt(data.total_requests) : 0; result.last_request = data.last_request ? data.last_request : null;