Skip to content
This repository has been archived by the owner on Apr 6, 2024. It is now read-only.

Commit

Permalink
Upload max of 5 active requests at a time
Browse files Browse the repository at this point in the history
Closes #9
  • Loading branch information
simonw committed Mar 20, 2024
1 parent 9f2a445 commit 0fe774c
Showing 1 changed file with 39 additions and 26 deletions.
65 changes: 39 additions & 26 deletions datasette_paste/templates/paste_create_table.html
Original file line number Diff line number Diff line change
Expand Up @@ -245,37 +245,23 @@ <h1>Paste data to create a table</h1>
contentTa.addEventListener('keyup', limited);
limited();

form.onsubmit = function(event) {
event.preventDefault();
const table = document.getElementById('id_table_name').value.trim();
if (!table) {
alert('Please enter a table name');
return;
}
const content = document.getElementById('id_content').value.trim();
if (!content) {
alert('Please paste some content');
return;
}
if (!currentRows.length) {
alert('No rows found in content');
return;
}
progressBar.style.display = 'block';
progressBar.value = 0;
progressBar.max = currentRows.length;
// Submit to API in batches of 100
const url = window.location.pathname.replace('/-/paste', '/-/create');
const headers = new Headers();
headers.append('Content-Type', 'application/json');
const batchSize = 100;
for (let i = 0; i < currentRows.length; i += batchSize) {
const batch = currentRows.slice(i, i + batchSize);
const batchSize = 100;
const numParallel = 5;

async function uploadRows(rows, url, headers, table, progressBar) {
let activeRequests = 0;
for (let i = 0; i < rows.length; i += batchSize) {
const batch = rows.slice(i, i + batchSize);
while (activeRequests >= numParallel) {
await new Promise(resolve => setTimeout(resolve, 25));
}
activeRequests++;
fetch(url, {
method: 'POST',
headers: headers,
body: JSON.stringify({table: table, rows: batch})
}).then(response => {
activeRequests--;
if (!response.ok) {
throw new Error('Failed to upload rows: ' + response.statusText);
}
Expand All @@ -290,9 +276,36 @@ <h1>Paste data to create a table</h1>
alert('Error uploading rows: ' + json.error);
}
}).catch(error => {
activeRequests--;
alert('Error uploading rows: ' + error);
});
}
}

form.onsubmit = function(event) {
event.preventDefault();
const table = document.getElementById('id_table_name').value.trim();
if (!table) {
alert('Please enter a table name');
return;
}
const content = document.getElementById('id_content').value.trim();
if (!content) {
alert('Please paste some content');
return;
}
if (!currentRows.length) {
alert('No rows found in content');
return;
}
progressBar.style.display = 'block';
progressBar.value = 0;
progressBar.max = currentRows.length;
// Submit to API in batches
const url = window.location.pathname.replace('/-/paste', '/-/create');
const headers = new Headers();
headers.append('Content-Type', 'application/json');
uploadRows(currentRows, url, headers, table, progressBar);
};

function parseJsonArray(string) {
Expand Down

0 comments on commit 0fe774c

Please sign in to comment.