Update meeting summaries retrieval and commit process

.github/workflows/commit-meeting-summaries.yml

@@ -1,27 +1,36 @@
 # .github/workflows/commit-meeting-summaries.yml
 name: Commit Meeting Summaries
-
 on:
   workflow_dispatch:
   schedule:
-    - cron: '0 0 * * *'  # Run daily at midnight
-
+    - cron: '0 0 * * *' # Run daily at midnight
 jobs:
   commit-meeting-summaries:
     runs-on: ubuntu-latest
-
     steps:
-      - name: Update GitHub Repository
+      - name: Update Meeting Summaries Array
         env:
           NETLIFY_BASE_URL: ${{ secrets.NETLIFY_BASE_URL }}
         run: |
-          response=$(curl -s -X POST -H "Content-Type: application/json" "${NETLIFY_BASE_URL}/.netlify/functions/updateGitHubRepo")
-          echo "Response from updateGitHubRepo: $response"
-
+          response=$(curl -s -X POST -H "Content-Type: application/json" "${NETLIFY_BASE_URL}/.netlify/functions/batchUpdateMeetingSummariesArray")
+          echo "Response from batchUpdateMeetingSummariesArray: $response"
           # Check if the response indicates success
           if echo "$response" | grep -qE '\{"message":\s*"Meeting summaries updated successfully"\}'; then
-            echo "Meeting summaries updated successfully"
+            echo "Meeting summaries array updated successfully"
           else
-            echo "Error updating meeting summaries"
+            echo "Error updating meeting summaries array"
+            exit 1
+          fi
+      - name: Update Meeting Summaries by ID
+        env:
+          NETLIFY_BASE_URL: ${{ secrets.NETLIFY_BASE_URL }}
+        run: |
+          response=$(curl -s -X POST -H "Content-Type: application/json" "${NETLIFY_BASE_URL}/.netlify/functions/batchUpdateMeetingSummariesById")
+          echo "Response from batchUpdateMeetingSummariesById: $response"
+          # Check if the response indicates success
+          if echo "$response" | grep -qE '\{"message":\s*"Meeting summaries updated successfully"\}'; then
+            echo "Meeting summaries by ID updated successfully"
+          else
+            echo "Error updating meeting summaries by ID"
             exit 1
           fi

netlify/functions/updateGitHubRepo.js → netlify/functions/batchUpdateMeetingSummariesArray.js

@@ -1,4 +1,4 @@
-// netlify/functions/updateGitHubRepo.js
+// netlify/functions/batchUpdateMeetingSummariesArray.js
 import { supabase } from '../../lib/supabaseClient';
 import { Octokit } from "@octokit/rest";
 
@@ -7,19 +7,19 @@ const BATCH_SIZE = 100;
 export const handler = async (event, context) => {
   try {
     let allSummaries = [];
-    let lastProcessedId = null;
+    let lastProcessedTimestamp = null;
     let hasMoreSummaries = true;
 
     while (hasMoreSummaries) {
       // Retrieve the next batch of summaries
       let { data: summaries, error } = await supabase
         .from('meetingsummaries')
-        .select('meeting_id, summary')
-        .order('meeting_id', { ascending: true })
+        .select('meeting_id, created_at, summary')
+        .order('created_at', { ascending: true })
         .limit(BATCH_SIZE);
 
-      if (lastProcessedId) {
-        summaries = summaries.filter(summary => summary.meeting_id > lastProcessedId);
+      if (lastProcessedTimestamp) {
+        summaries = summaries.filter(summary => summary.created_at > lastProcessedTimestamp);
       }
 
       if (error) {
@@ -37,7 +37,7 @@ export const handler = async (event, context) => {
 
       // Accumulate the summaries
       allSummaries = allSummaries.concat(summaries.map(summary => summary.summary));
-      lastProcessedId = summaries[summaries.length - 1].meeting_id;
+      lastProcessedTimestamp = summaries[summaries.length - 1].created_at;
     }
 
     // Commit all summaries to GitHub in a single file
@@ -51,7 +51,7 @@ export const handler = async (event, context) => {
       const { data: currentFile } = await octokit.repos.getContent({
         owner: "SingularityNET-Archive",
         repo: "SingularityNET-Archive",
-        path: "Data/meeting-summaries.json",
+        path: "Data/Meeting-Summaries/meeting-summaries-array.json",
       });
       currentSHA = currentFile.sha;
     } catch (error) {
@@ -63,7 +63,7 @@ export const handler = async (event, context) => {
     const { data } = await octokit.repos.createOrUpdateFileContents({
       owner: "SingularityNET-Archive",
       repo: "SingularityNET-Archive",
-      path: "Data/meeting-summaries.json",
+      path: "Data/Meeting-Summaries/meeting-summaries-array.json",
       message: "Update meeting summaries",
       content: Buffer.from(JSON.stringify(allSummaries, null, 2)).toString('base64'),
       sha: currentSHA,

netlify/functions/batchUpdateMeetingSummariesById.js

@@ -0,0 +1,87 @@
+// netlify/functions/batchUpdateMeetingSummariesById.js
+import { supabase } from '../../lib/supabaseClient';
+import { Octokit } from "@octokit/rest";
+
+const BATCH_SIZE = 200; // Increase the batch size
+const MAX_CONCURRENT_REQUESTS = 5; // Adjust the number of concurrent requests
+
+export const handler = async (event, context) => {
+  try {
+    let allSummaries = {};
+    let lastProcessedTimestamp = null;
+    let hasMoreSummaries = true;
+
+    while (hasMoreSummaries) {
+      const { data: summaries, error } = await supabase
+        .from('meetingsummaries')
+        .select('created_at, meeting_id, summary')
+        .order('created_at', { ascending: true })
+        .limit(BATCH_SIZE * MAX_CONCURRENT_REQUESTS)
+        .gt('created_at', lastProcessedTimestamp || '1970-01-01');
+
+      if (error) {
+        console.error('Error retrieving meeting summaries:', error);
+        return {
+          statusCode: 500,
+          body: JSON.stringify({ error: 'Failed to retrieve meeting summaries' }),
+        };
+      }
+
+      if (summaries.length === 0) {
+        hasMoreSummaries = false;
+        break;
+      }
+
+      // Group summaries by meeting_id
+      summaries.forEach(summary => {
+        const { meeting_id, summary: summaryText } = summary;
+        if (!allSummaries[meeting_id]) {
+          allSummaries[meeting_id] = [];
+        }
+        allSummaries[meeting_id].push(summaryText);
+      });
+
+      lastProcessedTimestamp = summaries[summaries.length - 1].created_at;
+    }
+
+    // Commit all summaries to GitHub in a single file
+    const octokit = new Octokit({
+      auth: process.env.GITHUB_TOKEN,
+    });
+
+    // Get the current SHA of the file
+    let currentSHA = null;
+    try {
+      const { data: currentFile } = await octokit.repos.getContent({
+        owner: "SingularityNET-Archive",
+        repo: "SingularityNET-Archive",
+        path: "Data/Meeting-Summaries/meeting-summaries-by-id.json",
+      });
+      currentSHA = currentFile.sha;
+    } catch (error) {
+      if (error.status !== 404) {
+        throw error;
+      }
+    }
+
+    const { data } = await octokit.repos.createOrUpdateFileContents({
+      owner: "SingularityNET-Archive",
+      repo: "SingularityNET-Archive",
+      path: "Data/Meeting-Summaries/meeting-summaries-by-id.json",
+      message: "Update meeting summaries",
+      content: Buffer.from(JSON.stringify(allSummaries, null, 2)).toString('base64'),
+      sha: currentSHA,
+    });
+
+    return {
+      statusCode: 200,
+      body: JSON.stringify({ message: 'Meeting summaries updated successfully' }),
+    };
+  } catch (error) {
+    console.error('Error in updateGitHubRepo function:', error);
+    return {
+      statusCode: 500,
+      body: JSON.stringify({ error: 'Failed to update meeting summaries' }),
+    };
+  }
+};