Skip to content

Commit d89f0dc

Browse files
Andre-DiamondAndre-Diamond
authored
Merge pull request #108 from SingularityNET-Archive:development
Refactor meeting summaries retrieval and commit process
2 parents 6e80160 + 2aa1cf3 commit d89f0dc

File tree

2 files changed

+161
-48
lines changed

2 files changed

+161
-48
lines changed

‎netlify/functions/batchUpdateMeetingSummariesArray.js‎

Lines changed: 95 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -2,65 +2,112 @@
22
import { supabase } from '../../lib/supabaseClient';
33
import { Octokit } from "@octokit/rest";
44

5-
export const handler = async (event, context) => {
6-
try {
7-
// Retrieve all summaries
8-
const { data: summaries, error } = await supabase
9-
.from('meetingsummaries')
10-
.select('meeting_id, created_at, summary')
11-
.eq('confirmed', true)
12-
.order('created_at', { ascending: true });
13-
14-
if (error) {
15-
console.error('Error retrieving meeting summaries:', error);
16-
return { statusCode: 500, body: JSON.stringify({ error: 'Failed to retrieve meeting summaries' }), };
5+
const BATCH_SIZE = 100;
6+
const MAX_CONCURRENT_REQUESTS = 10;
7+
8+
async function fetchMeetingSummaries(lastProcessedTimestamp, batchNumber) {
9+
const { data: summaries, error } = await supabase
10+
.from('meetingsummaries')
11+
.select('created_at, meeting_id, summary')
12+
.eq('confirmed', true)
13+
.order('created_at', { ascending: true })
14+
.limit(BATCH_SIZE)
15+
.gt('created_at', lastProcessedTimestamp || '1970-01-01')
16+
.range(batchNumber * BATCH_SIZE, (batchNumber + 1) * BATCH_SIZE - 1);
17+
18+
if (error) {
19+
throw new Error('Failed to retrieve meeting summaries');
20+
}
21+
22+
return summaries;
23+
}
24+
25+
function groupSummariesByYear(summaries, allSummaries) {
26+
summaries.forEach(summary => {
27+
const { summary: summaryText } = summary;
28+
const year = new Date(summaryText.meetingInfo.date).getFullYear();
29+
30+
if (!allSummaries[year]) {
31+
allSummaries[year] = [];
1732
}
1833

19-
// Group summaries by year
20-
const summariesByYear = {};
21-
summaries.forEach(summary => {
22-
const year = new Date(summary.summary.meetingInfo.date).getFullYear();
23-
if (!summariesByYear[year]) {
24-
summariesByYear[year] = [];
25-
}
26-
summariesByYear[year].push(summary.summary);
27-
});
34+
allSummaries[year].push(summaryText);
35+
});
36+
}
2837

29-
// Commit summaries to GitHub in separate year folders
30-
const octokit = new Octokit({ auth: process.env.GITHUB_TOKEN, });
31-
32-
for (const year in summariesByYear) {
33-
const yearSummaries = summariesByYear[year];
34-
const path = `Data/Meeting-Summaries/${year}/meeting-summaries-array.json`;
35-
36-
// Get the current SHA of the file
37-
let currentSHA = null;
38-
try {
39-
const { data: currentFile } = await octokit.repos.getContent({
40-
owner: "SingularityNET-Archive",
41-
repo: "SingularityNET-Archive",
42-
path,
43-
});
44-
currentSHA = currentFile.sha;
45-
} catch (error) {
46-
if (error.status !== 404) {
47-
throw error;
48-
}
49-
}
38+
async function commitSummariesToGitHub(allSummaries) {
39+
const octokit = new Octokit({ auth: process.env.GITHUB_TOKEN });
5040

51-
await octokit.repos.createOrUpdateFileContents({
41+
for (const year in allSummaries) {
42+
const yearSummaries = allSummaries[year];
43+
const path = `Data/Meeting-Summaries/${year}/meeting-summaries-array.json`;
44+
45+
let currentSHA = null;
46+
try {
47+
const { data: currentFile } = await octokit.repos.getContent({
5248
owner: "SingularityNET-Archive",
5349
repo: "SingularityNET-Archive",
5450
path,
55-
message: `Update meeting summaries for ${year}`,
56-
content: Buffer.from(JSON.stringify(yearSummaries, null, 2)).toString('base64'),
57-
sha: currentSHA,
5851
});
52+
currentSHA = currentFile.sha;
53+
} catch (error) {
54+
if (error.status !== 404) {
55+
throw error;
56+
}
57+
}
58+
59+
await octokit.repos.createOrUpdateFileContents({
60+
owner: "SingularityNET-Archive",
61+
repo: "SingularityNET-Archive",
62+
path,
63+
message: `Update meeting summaries for ${year}`,
64+
content: Buffer.from(JSON.stringify(yearSummaries, null, 2)).toString('base64'),
65+
sha: currentSHA,
66+
});
67+
}
68+
}
69+
70+
async function processAndCommitSummaries() {
71+
const allSummaries = {};
72+
let lastProcessedTimestamp = null;
73+
let hasMoreSummaries = true;
74+
let batchNumber = 0;
75+
76+
while (hasMoreSummaries) {
77+
const fetchPromises = [];
78+
for (let i = 0; i < MAX_CONCURRENT_REQUESTS; i++) {
79+
fetchPromises.push(fetchMeetingSummaries(lastProcessedTimestamp, batchNumber));
80+
batchNumber++;
81+
}
82+
83+
const summariesBatches = await Promise.all(fetchPromises);
84+
const flattenedSummaries = summariesBatches.flat();
85+
86+
if (flattenedSummaries.length === 0) {
87+
hasMoreSummaries = false;
88+
break;
5989
}
6090

61-
return { statusCode: 200, body: JSON.stringify({ message: 'Meeting summaries updated successfully' }), };
91+
groupSummariesByYear(flattenedSummaries, allSummaries);
92+
93+
lastProcessedTimestamp = flattenedSummaries[flattenedSummaries.length - 1].created_at;
94+
}
95+
96+
await commitSummariesToGitHub(allSummaries);
97+
}
98+
99+
export const handler = async (event, context) => {
100+
try {
101+
await processAndCommitSummaries();
102+
return {
103+
statusCode: 200,
104+
body: JSON.stringify({ message: 'Meeting summaries updated successfully' }),
105+
};
62106
} catch (error) {
63107
console.error('Error in updateGitHubRepo function:', error);
64-
return { statusCode: 500, body: JSON.stringify({ error: 'Failed to update meeting summaries' }), };
108+
return {
109+
statusCode: 500,
110+
body: JSON.stringify({ error: 'Failed to update meeting summaries' }),
111+
};
65112
}
66113
};

‎netlify/functions/singleCallUpdateMeetingSummariesArray.js‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
// netlify/functions/singleCallUpdateMeetingSummariesArray.js
2+
import { supabase } from '../../lib/supabaseClient';
3+
import { Octokit } from "@octokit/rest";
4+
5+
export const handler = async (event, context) => {
6+
try {
7+
// Retrieve all summaries
8+
const { data: summaries, error } = await supabase
9+
.from('meetingsummaries')
10+
.select('meeting_id, created_at, summary')
11+
.eq('confirmed', true)
12+
.order('created_at', { ascending: true });
13+
14+
if (error) {
15+
console.error('Error retrieving meeting summaries:', error);
16+
return { statusCode: 500, body: JSON.stringify({ error: 'Failed to retrieve meeting summaries' }), };
17+
}
18+
19+
// Group summaries by year
20+
const summariesByYear = {};
21+
summaries.forEach(summary => {
22+
const year = new Date(summary.summary.meetingInfo.date).getFullYear();
23+
if (!summariesByYear[year]) {
24+
summariesByYear[year] = [];
25+
}
26+
summariesByYear[year].push(summary.summary);
27+
});
28+
29+
// Commit summaries to GitHub in separate year folders
30+
const octokit = new Octokit({ auth: process.env.GITHUB_TOKEN, });
31+
32+
for (const year in summariesByYear) {
33+
const yearSummaries = summariesByYear[year];
34+
const path = `Data/Meeting-Summaries/${year}/meeting-summaries-array.json`;
35+
36+
// Get the current SHA of the file
37+
let currentSHA = null;
38+
try {
39+
const { data: currentFile } = await octokit.repos.getContent({
40+
owner: "SingularityNET-Archive",
41+
repo: "SingularityNET-Archive",
42+
path,
43+
});
44+
currentSHA = currentFile.sha;
45+
} catch (error) {
46+
if (error.status !== 404) {
47+
throw error;
48+
}
49+
}
50+
51+
await octokit.repos.createOrUpdateFileContents({
52+
owner: "SingularityNET-Archive",
53+
repo: "SingularityNET-Archive",
54+
path,
55+
message: `Update meeting summaries for ${year}`,
56+
content: Buffer.from(JSON.stringify(yearSummaries, null, 2)).toString('base64'),
57+
sha: currentSHA,
58+
});
59+
}
60+
61+
return { statusCode: 200, body: JSON.stringify({ message: 'Meeting summaries updated successfully' }), };
62+
} catch (error) {
63+
console.error('Error in updateGitHubRepo function:', error);
64+
return { statusCode: 500, body: JSON.stringify({ error: 'Failed to update meeting summaries' }), };
65+
}
66+
};

0 commit comments

Comments
 (0)