Update meeting summaries retrieval and commit process

.github/workflows/commit-meeting-summaries.yml

@@ -11,39 +11,17 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - name: Install dependencies
-        run: |
-          sudo apt-get install -y jq
-          # Install any other necessary dependencies
-
-      - name: Retrieve Meeting Summaries
-        id: retrieve-summaries
+      - name: Update GitHub Repository
         env:
           NETLIFY_BASE_URL: ${{ secrets.NETLIFY_BASE_URL }}
         run: |
-          # Make a call to your getMeetingSummaries Netlify function
-          response=$(curl -s -X POST -H "Content-Type: application/json" -d '{}' "${NETLIFY_BASE_URL}/.netlify/functions/getMeetingSummaries")
-          echo "Response from getMeetingSummaries: $response"
+          response=$(curl -s -X POST -H "Content-Type: application/json" "${NETLIFY_BASE_URL}/.netlify/functions/updateGitHubRepo")
+          echo "Response from updateGitHubRepo: $response"
 
-          # Check if the response is valid JSON
-          if ! echo "$response" | jq -e . >/dev/null 2>&1; then
-            echo "Error: Invalid JSON response from getMeetingSummaries function"
+          # Check if the response indicates success
+          if echo "$response" | grep -q '"message": "Meeting summaries updated successfully"'; then
+            echo "Meeting summaries updated successfully"
+          else
+            echo "Error updating meeting summaries"
             exit 1
-          fi
-
-          summaries=$(echo "$response" | jq -c '.')
-          echo "summaries=$summaries" >> $GITHUB_OUTPUT
-
-      - name: Commit to GitHub
-        env:
-          NETLIFY_BASE_URL: ${{ secrets.NETLIFY_BASE_URL }}
-        run: |
-          curl -X POST "${NETLIFY_BASE_URL}/.netlify/functions/commitToGitHub" \
-            -H 'Content-Type: application/json' \
-            -d '{
-              "owner": "SingularityNET-Archive",
-              "repo": "SingularityNET-Archive",
-              "filePath": "Data/meeting-summaries.json",
-              "content": ${{ steps.retrieve-summaries.outputs.summaries }},
-              "commitMessage": "Update meeting summaries"
-            }'
+          fi

netlify/functions/updateGitHubRepo.js

@@ -0,0 +1,64 @@
+// netlify/functions/updateGitHubRepo.js
+import { supabase } from '../../lib/supabaseClient';
+import { Octokit } from "@octokit/rest";
+
+const BATCH_SIZE = 100;
+
+export const handler = async (event, context) => {
+  try {
+    let allSummaries = [];
+    let lastProcessedId = null;
+    let hasMoreSummaries = true;
+
+    while (hasMoreSummaries) {
+      // Retrieve the next batch of summaries
+      const { data: summaries, error } = await supabase
+        .from('meetingsummaries')
+        .select('meeting_id, summary')
+        .gt('meeting_id', lastProcessedId || 0)
+        .limit(BATCH_SIZE)
+        .order('meeting_id', { ascending: true });
+
+      if (error) {
+        console.error('Error retrieving meeting summaries:', error);
+        return {
+          statusCode: 500,
+          body: JSON.stringify({ error: 'Failed to retrieve meeting summaries' }),
+        };
+      }
+
+      if (summaries.length === 0) {
+        hasMoreSummaries = false;
+        break;
+      }
+
+      // Accumulate the summaries
+      allSummaries = allSummaries.concat(summaries);
+      lastProcessedId = summaries[summaries.length - 1].meeting_id;
+    }
+
+    // Commit all summaries to GitHub in a single file
+    const octokit = new Octokit({
+      auth: process.env.GITHUB_TOKEN,
+    });
+
+    const { data } = await octokit.repos.createOrUpdateFileContents({
+      owner: "SingularityNET-Archive",
+      repo: "SingularityNET-Archive",
+      path: "Data/meeting-summaries.json",
+      message: "Update meeting summaries",
+      content: Buffer.from(JSON.stringify(allSummaries, null, 2)).toString('base64'),
+    });
+
+    return {
+      statusCode: 200,
+      body: JSON.stringify({ message: 'Meeting summaries updated successfully' }),
+    };
+  } catch (error) {
+    console.error('Error in updateGitHubRepo function:', error);
+    return {
+      statusCode: 500,
+      body: JSON.stringify({ error: 'Failed to update meeting summaries' }),
+    };
+  }
+};