From e0670d65d3c2e19c5174962d7e27b6fea4f016fa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Andr=C3=A9=20Diamond?=
 <32074058+Andre-Diamond@users.noreply.github.com>
Date: Tue, 30 Apr 2024 07:58:07 +0200
Subject: [PATCH] Refactor meeting summaries retrieval and commit process

---
 .../batchUpdateMeetingSummariesArray.js       | 76 ++++++++++---------
 .../batchUpdateMeetingSummariesById.js        | 55 +++++++++-----
 2 files changed, 73 insertions(+), 58 deletions(-)

diff --git a/netlify/functions/batchUpdateMeetingSummariesArray.js b/netlify/functions/batchUpdateMeetingSummariesArray.js
index 9abb592..21070f3 100644
--- a/netlify/functions/batchUpdateMeetingSummariesArray.js
+++ b/netlify/functions/batchUpdateMeetingSummariesArray.js
@@ -8,57 +8,59 @@ export const handler = async (event, context) => {
     const { data: summaries, error } = await supabase
       .from('meetingsummaries')
       .select('meeting_id, created_at, summary')
+      .eq('confirmed', true)
       .order('created_at', { ascending: true });
 
     if (error) {
       console.error('Error retrieving meeting summaries:', error);
-      return {
-        statusCode: 500,
-        body: JSON.stringify({ error: 'Failed to retrieve meeting summaries' }),
-      };
+      return { statusCode: 500, body: JSON.stringify({ error: 'Failed to retrieve meeting summaries' }), };
     }
 
-    // Extract summaries from the retrieved data
-    const allSummaries = summaries.map(summary => summary.summary);
-
-    // Commit all summaries to GitHub in a single file
-    const octokit = new Octokit({
-      auth: process.env.GITHUB_TOKEN,
+    // Group summaries by year
+    const summariesByYear = {};
+    summaries.forEach(summary => {
+      const year = new Date(summary.summary.meetingInfo.date).getFullYear();
+      if (!summariesByYear[year]) {
+        summariesByYear[year] = [];
+      }
+      summariesByYear[year].push(summary.summary);
     });
 
-    // Get the current SHA of the file
-    let currentSHA = null;
-    try {
-      const { data: currentFile } = await octokit.repos.getContent({
+    // Commit summaries to GitHub in separate year folders
+    const octokit = new Octokit({ auth: process.env.GITHUB_TOKEN, });
+
+    for (const year in summariesByYear) {
+      const yearSummaries = summariesByYear[year];
+      const path = `Data/Meeting-Summaries/${year}/meeting-summaries-array.json`;
+
+      // Get the current SHA of the file
+      let currentSHA = null;
+      try {
+        const { data: currentFile } = await octokit.repos.getContent({
+          owner: "SingularityNET-Archive",
+          repo: "SingularityNET-Archive",
+          path,
+        });
+        currentSHA = currentFile.sha;
+      } catch (error) {
+        if (error.status !== 404) {
+          throw error;
+        }
+      }
+
+      await octokit.repos.createOrUpdateFileContents({
         owner: "SingularityNET-Archive",
         repo: "SingularityNET-Archive",
-        path: "Data/Meeting-Summaries/meeting-summaries-array.json",
+        path,
+        message: `Update meeting summaries for ${year}`,
+        content: Buffer.from(JSON.stringify(yearSummaries, null, 2)).toString('base64'),
+        sha: currentSHA,
       });
-      currentSHA = currentFile.sha;
-    } catch (error) {
-      if (error.status !== 404) {
-        throw error;
-      }
     }
 
-    const { data } = await octokit.repos.createOrUpdateFileContents({
-      owner: "SingularityNET-Archive",
-      repo: "SingularityNET-Archive",
-      path: "Data/Meeting-Summaries/meeting-summaries-array.json",
-      message: "Update meeting summaries",
-      content: Buffer.from(JSON.stringify(allSummaries, null, 2)).toString('base64'),
-      sha: currentSHA,
-    });
-
-    return {
-      statusCode: 200,
-      body: JSON.stringify({ message: 'Meeting summaries updated successfully' }),
-    };
+    return { statusCode: 200, body: JSON.stringify({ message: 'Meeting summaries updated successfully' }), };
   } catch (error) {
     console.error('Error in updateGitHubRepo function:', error);
-    return {
-      statusCode: 500,
-      body: JSON.stringify({ error: 'Failed to update meeting summaries' }),
-    };
+    return { statusCode: 500, body: JSON.stringify({ error: 'Failed to update meeting summaries' }), };
   }
 };
\ No newline at end of file
diff --git a/netlify/functions/batchUpdateMeetingSummariesById.js b/netlify/functions/batchUpdateMeetingSummariesById.js
index b138587..5bbc109 100644
--- a/netlify/functions/batchUpdateMeetingSummariesById.js
+++ b/netlify/functions/batchUpdateMeetingSummariesById.js
@@ -9,6 +9,7 @@ async function fetchMeetingSummaries(lastProcessedTimestamp) {
   const { data: summaries, error } = await supabase
     .from('meetingsummaries')
     .select('created_at, meeting_id, summary')
+    .eq('confirmed', true)
     .order('created_at', { ascending: true })
     .limit(BATCH_SIZE * MAX_CONCURRENT_REQUESTS)
     .gt('created_at', lastProcessedTimestamp || '1970-01-01');
@@ -23,38 +24,50 @@ async function fetchMeetingSummaries(lastProcessedTimestamp) {
 function groupSummariesByMeetingId(summaries, allSummaries) {
   summaries.forEach(summary => {
     const { meeting_id, summary: summaryText } = summary;
-    if (!allSummaries[meeting_id]) {
-      allSummaries[meeting_id] = [];
+    const year = new Date(summaryText.meetingInfo.date).getFullYear();
+
+    if (!allSummaries[year]) {
+      allSummaries[year] = {};
+    }
+
+    if (!allSummaries[year][meeting_id]) {
+      allSummaries[year][meeting_id] = [];
     }
-    allSummaries[meeting_id].push(summaryText);
+
+    allSummaries[year][meeting_id].push(summaryText);
   });
 }
 
 async function commitSummariesToGitHub(allSummaries) {
   const octokit = new Octokit({ auth: process.env.GITHUB_TOKEN });
 
-  let currentSHA = null;
-  try {
-    const { data: currentFile } = await octokit.repos.getContent({
+  for (const year in allSummaries) {
+    const yearSummaries = allSummaries[year];
+    const path = `Data/Meeting-Summaries/${year}/meeting-summaries-by-id.json`;
+
+    let currentSHA = null;
+    try {
+      const { data: currentFile } = await octokit.repos.getContent({
+        owner: "SingularityNET-Archive",
+        repo: "SingularityNET-Archive",
+        path,
+      });
+      currentSHA = currentFile.sha;
+    } catch (error) {
+      if (error.status !== 404) {
+        throw error;
+      }
+    }
+
+    await octokit.repos.createOrUpdateFileContents({
       owner: "SingularityNET-Archive",
       repo: "SingularityNET-Archive",
-      path: "Data/Meeting-Summaries/meeting-summaries-by-id.json",
+      path,
+      message: `Update meeting summaries for ${year}`,
+      content: Buffer.from(JSON.stringify(yearSummaries, null, 2)).toString('base64'),
+      sha: currentSHA,
     });
-    currentSHA = currentFile.sha;
-  } catch (error) {
-    if (error.status !== 404) {
-      throw error;
-    }
   }
-
-  await octokit.repos.createOrUpdateFileContents({
-    owner: "SingularityNET-Archive",
-    repo: "SingularityNET-Archive",
-    path: "Data/Meeting-Summaries/meeting-summaries-by-id.json",
-    message: "Update meeting summaries",
-    content: Buffer.from(JSON.stringify(allSummaries, null, 2)).toString('base64'),
-    sha: currentSHA,
-  });
 }
 
 async function processAndCommitSummaries() {