Merge pull request #12 from Sentiopulse/feat/add-upstash-int

tasin2610 · web-flow · commit 859157276647 · 2025-09-17T01:23:08.000-06:00
feat: Integrate Upstash Redis for Dynamic PostGroup Storage and Retrieval
diff --git a/package.json b/package.json
@@ -11,7 +11,10 @@
     "classify:title": "tsx run-classifier.ts title",
     "dev": "tsx --watch run-classifier.ts",
     "build": "tsc",
-    "start": "node dist/run-classifier.js"
+    "start": "node dist/run-classifier.js",
+    "seed": "tsx src/seed.ts",
+    "seed:clear": "tsx src/seed.ts clear",
+    "seed:verify": "tsx src/seed.ts verify"
   },
   "dependencies": {
     "@prisma/client": "^6.15.0",
diff --git a/src/postGroup.ts b/src/postGroup.ts
@@ -1,5 +1,7 @@
+
 import cron from 'node-cron';
 import { generateTitleForPost } from './generateTitle';
+import { initRedis, getRedisClient } from './redisClient';
 
 export type Post = {
     id: string;
@@ -25,62 +27,40 @@ export async function generateTitleForPostGroup(postGroup: PostGroup): Promise<s
     return await generateTitleForPost(combinedContent);
 }
 
-//One group with several posts
-const postGroups: PostGroup[] = [
-    {
-        id: "group1",
-        posts: [
-            {
-                id: "post1",
-                content: `Empery Digital\n@EMPD_BTC\n·\n11m\nBitcoin Firsts that changed everything:\n- $4B Pizza\n- A nation bets on BTC\n- Wall Street embraces it\n- The Trillion-Dollar Club\nFrom a pizza order to reshaping global finance.\n#Bitcoin #BTC #Blockchain #EmperyDigital`,
-                sentiment: "BULLISH",
-                source: "TWITTER",
-                categories: ["Cryptocurrency", "Market Analysis"],
-                subcategories: ["Bitcoin", "Milestones", "Adoption"],
-                link: undefined,
-                createdAt: new Date().toISOString(),
-                updatedAt: new Date().toISOString(),
-            },
-            {
-                id: "post2",
-                content: `Empery Digital\n@EMPD_BTC\n·\n11m\nSome notable events in Bitcoin's history include:\n- The purchase of pizza with Bitcoin\n- A country adopting BTC\n- Increased interest from Wall Street\n- Joining the Trillion-Dollar Club\nThese milestones reflect Bitcoin's evolving role in finance.\n#Bitcoin #BTC #Blockchain #EmperyDigital`,
-                sentiment: "NEUTRAL",
-                source: "TWITTER",
-                categories: ["Cryptocurrency", "Market Analysis"],
-                subcategories: ["Bitcoin", "Milestones", "Adoption"],
-                link: undefined,
-                createdAt: new Date().toISOString(),
-                updatedAt: new Date().toISOString(),
-            },
-            {
-                id: "post3",
-                content: `Empery Digital\n@EMPD_BTC\n·\n11m\nRecent events in Bitcoin's history have raised concerns:\n- The infamous $4B pizza purchase\n- A nation risking its economy on BTC\n- Wall Street's speculative involvement\n- Entering the Trillion-Dollar Club amid volatility\nFrom a simple transaction to ongoing financial uncertainty.\n#Bitcoin #BTC #Blockchain #EmperyDigital`,
-                sentiment: "BEARISH",
-                source: "TWITTER",
-                categories: ["Cryptocurrency", "Market Analysis"],
-                subcategories: ["Bitcoin", "Milestones", "Risks"],
-                link: undefined,
-                createdAt: new Date().toISOString(),
-                updatedAt: new Date().toISOString(),
-            }
-        ]
-    }
-];
 
+// Fetch all PostGroups from Redis (expects a key 'PostGroup' with a JSON array, or adapt as needed)
+export async function fetchPostGroupsFromRedis(): Promise<PostGroup[]> {
+    await initRedis();
+    const redis = getRedisClient();
+    // Adjust the key if you use a different one
+    const data = await redis.get('post-groups');
+    if (!data) return [];
+    try {
+        return JSON.parse(data);
+    } catch (e) {
+        console.error('Failed to parse post-groups data from Redis:', e);
+        return [];
+    }
+}
 
-export default postGroups;
 
 
-// Reusable function to generate and log the title for the first PostGroup
+// Reusable function to generate and log the title for all PostGroups from Redis
 export async function logTitlesForAllPostGroups(context: 'CRON' | 'MANUAL' = 'MANUAL') {
+    const postGroups = await fetchPostGroupsFromRedis();
     if (!postGroups.length) {
-        console.log('No PostGroups found.');
+        console.log('No PostGroups found in Redis.');
         return;
     }
+    let updated = false;
+    const postGroupsWithOrderedKeys = [];
     for (const group of postGroups) {
+        let title = group.title;
         try {
-            const title = await generateTitleForPostGroup(group);
-            group.title = title;
+            title = await generateTitleForPostGroup(group);
+            if (group.title !== title) {
+                updated = true;
+            }
             if (context === 'CRON') {
                 console.log(`[CRON] Generated Title for PostGroup (id: ${group.id}) at ${new Date().toISOString()}:`, title);
             } else {
@@ -93,6 +73,22 @@ export async function logTitlesForAllPostGroups(context: 'CRON' | 'MANUAL' = 'MA
                 console.error(`Error generating title for PostGroup (id: ${group.id}):`, e);
             }
         }
+        postGroupsWithOrderedKeys.push({
+            id: group.id,
+            title,
+            posts: group.posts
+        });
+    }
+    // Save updated PostGroups with titles back to Redis
+    if (updated) {
+        await initRedis();
+        const redis = getRedisClient();
+        await redis.set('post-groups', JSON.stringify(postGroupsWithOrderedKeys));
+        if (context === 'CRON') {
+            console.log('[CRON] Updated post-groups with titles saved to Redis.');
+        } else {
+            console.log('Updated post-groups with titles saved to Redis.');
+        }
     }
 }
 
diff --git a/src/redisClient.ts b/src/redisClient.ts
@@ -51,7 +51,6 @@ export async function initRedis(): Promise<RedisClientType> {
         await newClient.connect();
         // assign to module client (cast to satisfy TS) and return the concrete client
         client = newClient as unknown as RedisClientType;
-        console.log("Redis connected:", safeRedisUrlForLog(REDIS_URL));
         // Clear connecting before returning so subsequent callers don't wait
         connecting = null;
         return newClient as unknown as RedisClientType;
diff --git a/src/seed.ts b/src/seed.ts
@@ -0,0 +1,85 @@
+#!/usr/bin/env node
+
+import { seedDatabase, clearDatabase, verifySeeding } from './seedDatabase';
+
+/**
+ * Command-line script to seed the Upstash Redis database with mock data.
+ * 
+ * Usage:
+ *   npx tsx src/seed.ts              # Seed the database
+ *   npx tsx src/seed.ts clear        # Clear all PostGroup data
+ *   npx tsx src/seed.ts verify       # Verify seeded data
+ *   npx tsx src/seed.ts --help       # Show help
+ */
+
+async function main() {
+    const args = process.argv.slice(2);
+    const command = args[0]?.toLowerCase();
+
+    try {
+        switch (command) {
+            case 'clear':
+                await clearDatabase();
+                break;
+
+            case 'verify':
+                await verifySeeding();
+                break;
+
+            case '--help':
+            case '-h':
+            case 'help':
+                showHelp();
+                break;
+
+            case undefined:
+                // Default action: seed the database
+                await seedDatabase();
+                await verifySeeding(); // Verify after seeding
+                break;
+
+            default:
+                console.error(`❌ Unknown command: ${command}`);
+                showHelp();
+                process.exit(1);
+        }
+
+        console.log('✨ Operation completed successfully!');
+        process.exit(0);
+
+    } catch (error) {
+        console.error('💥 Operation failed:', error);
+        process.exit(1);
+    }
+}
+
+function showHelp() {
+    console.log(`
+🌱 Database Seeding Script
+
+Usage:
+  npx tsx src/seed.ts              # Seed the database with mock data
+  npx tsx src/seed.ts clear        # Clear all PostGroup data (use with caution!)
+  npx tsx src/seed.ts verify       # Verify that data was seeded correctly
+  npx tsx src/seed.ts --help       # Show this help message
+
+Examples:
+  # Seed the database with mock post groups and posts
+  npx tsx src/seed.ts
+
+  # Clear all data before seeding fresh data
+  npx tsx src/seed.ts clear && npx tsx src/seed.ts
+
+  # Just verify what's currently in the database
+  npx tsx src/seed.ts verify
+
+Environment Requirements:
+  - REDIS_URL or Upstash environment variables must be configured
+  - See .env file for configuration details
+`);
+}
+
+// Run the script
+if (require.main === module) {
+    main();
+}
diff --git a/src/seedData.ts b/src/seedData.ts
@@ -0,0 +1,60 @@
+import { PostGroup } from './postGroup';
+
+export const seedData: PostGroup[] = [
+    {
+        "id": "group1",
+        "posts": [
+            {
+                "id": "post1",
+                "content": "Empery Digital\n@EMPD_BTC\n·\n11m\nBitcoin Firsts that changed everything:\n- $4B Pizza\n- A nation bets on BTC\n- Wall Street embraces it\n- The Trillion-Dollar Club\nFrom a pizza order to reshaping global finance.\n#Bitcoin #BTC #Blockchain #EmperyDigital",
+                "sentiment": "BULLISH",
+                "source": "TWITTER",
+                "categories": [
+                    "Cryptocurrency",
+                    "Market Analysis"
+                ],
+                "subcategories": [
+                    "Bitcoin",
+                    "Milestones",
+                    "Adoption"
+                ],
+                "createdAt": "2025-09-16T12:00:00.000Z",
+                "updatedAt": "2025-09-16T12:00:00.000Z"
+            },
+            {
+                "id": "post2",
+                "content": "Empery Digital\n@EMPD_BTC\n·\n11m\nSome notable events in Bitcoin's history include:\n- The purchase of pizza with Bitcoin\n- A country adopting BTC\n- Increased interest from Wall Street\n- Joining the Trillion-Dollar Club\nThese milestones reflect Bitcoin's evolving role in finance.\n#Bitcoin #BTC #Blockchain #EmperyDigital",
+                "sentiment": "NEUTRAL",
+                "source": "TWITTER",
+                "categories": [
+                    "Cryptocurrency",
+                    "Market Analysis"
+                ],
+                "subcategories": [
+                    "Bitcoin",
+                    "Milestones",
+                    "Adoption"
+                ],
+                "createdAt": "2025-09-16T12:00:00.000Z",
+                "updatedAt": "2025-09-16T12:00:00.000Z"
+            },
+            {
+                "id": "post3",
+                "content": "Empery Digital\n@EMPD_BTC\n·\n11m\nRecent events in Bitcoin's history have raised concerns:\n- The infamous $4B pizza purchase\n- A nation risking its economy on BTC\n- Wall Street's speculative involvement\n- Entering the Trillion-Dollar Club amid volatility\nFrom a simple transaction to ongoing financial uncertainty.\n#Bitcoin #BTC #Blockchain #EmperyDigital",
+                "sentiment": "BEARISH",
+                "source": "TWITTER",
+                "categories": [
+                    "Cryptocurrency",
+                    "Market Analysis"
+                ],
+                "subcategories": [
+                    "Bitcoin",
+                    "Milestones",
+                    "Risks"
+                ],
+                "createdAt": "2025-09-16T12:00:00.000Z",
+                "updatedAt": "2025-09-16T12:00:00.000Z"
+            }
+        ]
+    }
+];
diff --git a/src/seedDatabase.ts b/src/seedDatabase.ts
@@ -0,0 +1,90 @@
+import { initRedis, getRedisClient } from './redisClient';
+import { seedData } from './seedData';
+import { PostGroup } from './postGroup';
+
+/**
+ * Seeds the Upstash Redis database with mock data for PostGroups.
+ * This function will clear existing PostGroup data and populate it with the seed data.
+ */
+export async function seedDatabase(): Promise<void> {
+    try {
+        console.log('🌱 Starting database seeding...');
+
+        // Initialize Redis connection
+        await initRedis();
+        const redis = getRedisClient();
+
+        console.log('✅ Connected to Redis');
+
+        // Store the seed data as post-groups data in Redis
+        // Using the new key structure as required
+        await redis.set('post-groups', JSON.stringify(seedData));
+
+        console.log(`✅ Successfully seeded ${seedData.length} post group(s) to Redis`);
+        console.log('📊 Seed data summary:');
+
+        seedData.forEach((group: PostGroup, index: number) => {
+            console.log(`   Group ${index + 1}: ${group.id} (${group.posts.length} posts)`);
+            group.posts.forEach((post, postIndex) => {
+                console.log(`     Post ${postIndex + 1}: ${post.id} - ${post.sentiment} (${post.source})`);
+            });
+        });
+
+        console.log('🎉 Database seeding completed successfully!');
+
+    } catch (error) {
+        console.error('❌ Error seeding database:', error);
+        throw error;
+    }
+}
+
+/**
+ * Clears all PostGroup data from the database.
+ * Use with caution - this will delete all existing post groups!
+ */
+export async function clearDatabase(): Promise<void> {
+    try {
+        console.log('🧹 Clearing PostGroup data from database...');
+
+        await initRedis();
+        const redis = getRedisClient();
+
+        await redis.del('post-groups');
+
+        console.log('✅ Database cleared successfully');
+
+    } catch (error) {
+        console.error('❌ Error clearing database:', error);
+        throw error;
+    }
+}
+
+/**
+ * Verifies that the seed data was properly stored by retrieving and logging it.
+ */
+export async function verifySeeding(): Promise<void> {
+    try {
+        console.log('🔍 Verifying seeded data...');
+
+        await initRedis();
+        const redis = getRedisClient();
+
+        const data = await redis.get('post-groups');
+
+        if (!data) {
+            console.log('❌ No data found in database');
+            return;
+        }
+
+        const postGroups: PostGroup[] = JSON.parse(data);
+
+        console.log(`✅ Found ${postGroups.length} post group(s) in database:`);
+        postGroups.forEach((group, index) => {
+            console.log(`   Group ${index + 1}: ${group.id} (${group.posts.length} posts)`);
+        });
+
+    } catch (error) {
+        console.error('❌ Error verifying seeded data:', error);
+        throw error;
+    }
+}
