Skip to content
Permalink
Browse files

Implemented parallel URL uploads

This doesn't use the server's built-in ability to accept multiple URLs
per API request.
It behaves the same as regular uploads, in that it executes one API call
per file, simultaneously.

I figured this is a better implementation to shift queues faster.

---

Fetch error from URL uploads due to exceeding size limit will no longer
be logged in server's console.

Clients will also see better formatted error message for URL uploads'
file size limit errors.

---

Bumped dependencies:
knex: 0.20.2 -> 0.20.3
systeminformation: 4.15.3 -> 4.16.0

Bumped v1 version string
  • Loading branch information
BobbyWibowo committed Nov 29, 2019
1 parent df1e835 commit 337a0a61ff13dab2ec7f488218faf100ce225727
Showing with 122 additions and 102 deletions.
  1. +1 −1 TODO.md
  2. +8 −2 controllers/uploadController.js
  3. +1 −1 dist/js/home.js
  4. +1 −1 dist/js/home.js.map
  5. +2 −2 package.json
  6. +82 −68 src/js/home.js
  7. +1 −1 src/versions.json
  8. +26 −26 yarn.lock
@@ -35,7 +35,7 @@ Consider remembering last pages of each individual albums as well. When deleting

Low priority:

* [ ] Parallel URL uploads.
* [x] Parallel URL uploads.
* [x] Delete user feature.
* [ ] Bulk delete user feature.
* [ ] Bulk disable user feature.
@@ -363,8 +363,14 @@ self.actuallyUploadUrls = async (req, res, user, albumid, age) => {
utils.unlinkFile(file).catch(logger.error)
))

// Re-throw error
throw error
const errorString = error.toString()
const suppress = [
/ over limit:/
]
if (!suppress.some(t => t.test(errorString)))
throw error
else
throw errorString
}
}

Large diffs are not rendered by default.

Large diffs are not rendered by default.

@@ -37,15 +37,15 @@
"fluent-ffmpeg": "^2.1.2",
"helmet": "^3.21.2",
"jszip": "^3.2.2",
"knex": "^0.20.2",
"knex": "^0.20.3",
"multer": "^1.4.2",
"node-fetch": "^2.6.0",
"nunjucks": "^3.2.0",
"randomstring": "^1.1.5",
"readline": "^1.3.0",
"sharp": "^0.23.3",
"sqlite3": "^4.1.0",
"systeminformation": "^4.15.3"
"systeminformation": "^4.16.0"
},
"devDependencies": {
"browserslist": "^4.7.3",
@@ -43,6 +43,10 @@ const page = {
clipboardJS: null,
lazyLoad: null,

// additional vars for url uploads
urlsQueue: [],
activeUrlsQueue: 0,

// Include BMP for uploads preview only, cause the real images will be used
// Sharp isn't capable of making their thumbnails for dashboard and album public pages
imageExts: ['.webp', '.jpg', '.jpeg', '.bmp', '.gif', '.png', '.tiff', '.tif', '.svg'],
@@ -215,7 +219,7 @@ page.prepareUpload = () => {
page.urlMaxSizeBytes = page.urlMaxSize * 1e6
urlMaxSize.innerHTML = page.getPrettyBytes(page.urlMaxSizeBytes)
document.querySelector('#uploadUrls').addEventListener('click', event => {
page.uploadUrls(event.currentTarget)
page.addUrlsToQueue()
})
}

@@ -382,17 +386,17 @@ page.prepareDropzone = () => {
`${prefix} ${percentage}%${prettyBytesPerSec ? ` at ~${prettyBytesPerSec}/s` : ''}`
})

this.on('success', (file, response) => {
if (!response) return
this.on('success', (file, data) => {
if (!data) return
file.previewElement.querySelector('.descriptive-progress').classList.add('is-hidden')

if (response.success === false) {
file.previewElement.querySelector('.error').innerHTML = response.description
if (data.success === false) {
file.previewElement.querySelector('.error').innerHTML = data.description
file.previewElement.querySelector('.error').classList.remove('is-hidden')
}

if (response.files && response.files[0])
page.updateTemplate(file, response.files[0])
if (Array.isArray(data.files) && data.files[0])
page.updateTemplate(file, data.files[0])
})

this.on('error', (file, error) => {
@@ -404,7 +408,6 @@ page.prepareDropzone = () => {
page.updateTemplateIcon(file.previewElement, 'icon-block')

file.previewElement.querySelector('.descriptive-progress').classList.add('is-hidden')
file.previewElement.querySelector('.name').innerHTML = file.name

file.previewElement.querySelector('.error').innerHTML = error.description || error
file.previewElement.querySelector('.error').classList.remove('is-hidden')
@@ -454,85 +457,96 @@ page.prepareDropzone = () => {
})
}

page.uploadUrls = button => {
const tabDiv = document.querySelector('#tab-urls')
if (!tabDiv || button.classList.contains('is-loading'))
return
page.addUrlsToQueue = () => {
const urls = document.querySelector('#urls').value
.split(/\r?\n/)
.filter(url => {
return url.trim().length
})

button.classList.add('is-loading')
if (!urls.length)
return swal('An error occurred!', 'You have not entered any URLs.', 'error')

function done (error) {
if (error) swal('An error occurred!', error, 'error')
button.classList.remove('is-loading')
}
const tabDiv = document.querySelector('#tab-urls')
tabDiv.querySelector('.uploads').classList.remove('is-hidden')

function run () {
const headers = {
token: page.token,
albumid: page.album,
age: page.uploadAge,
filelength: page.fileLength
}
for (let i = 0; i < urls.length; i++) {
const previewTemplate = document.createElement('template')
previewTemplate.innerHTML = page.previewTemplate.trim()

const previewElement = previewTemplate.content.firstChild
previewElement.querySelector('.name').innerHTML = urls[i]
previewElement.querySelector('.descriptive-progress').innerHTML = 'Waiting in queue\u2026'

const previewsContainer = tabDiv.querySelector('.uploads')
previewsContainer.appendChild(previewElement)

const urls = document.querySelector('#urls').value
.split(/\r?\n/)
.filter(url => {
return url.trim().length
})
document.querySelector('#urls').value = urls.join('\n')
page.urlsQueue.push({
url: urls[i],
previewElement
})
}

if (!urls.length)
return done('You have not entered any URLs.')
page.processUrlsQueue()
document.querySelector('#urls').value = ''
}

tabDiv.querySelector('.uploads').classList.remove('is-hidden')
page.processUrlsQueue = () => {
if (!page.urlsQueue.length) return

const files = urls.map(url => {
const previewTemplate = document.createElement('template')
previewTemplate.innerHTML = page.previewTemplate.trim()
function finishedUrlUpload (file, data) {
file.previewElement.querySelector('.descriptive-progress').classList.add('is-hidden')

const previewElement = previewTemplate.content.firstChild
previewElement.querySelector('.name').innerHTML = url
previewElement.querySelector('.descriptive-progress').innerHTML = 'Waiting in queue\u2026'
if (data.success === false) {
const match = data.description.match(/ over limit: (\d+)$/)
if (match && match[1])
data.description = `File exceeded limit of ${page.getPrettyBytes(match[1])}.`

previewsContainer.appendChild(previewElement)
return { url, previewElement }
})
file.previewElement.querySelector('.error').innerHTML = data.description
file.previewElement.querySelector('.error').classList.remove('is-hidden')
}

function post (i) {
if (i === files.length)
return done()
if (Array.isArray(data.files) && data.files[0])
page.updateTemplate(file, data.files[0])

function posted (result) {
files[i].previewElement.querySelector('.descriptive-progress').classList.add('is-hidden')
page.activeUrlsQueue--
return shiftQueue()
}

if (result.success) {
page.updateTemplate(files[i], result.files[0])
} else {
page.updateTemplateIcon(files[i].previewElement, 'icon-block')
files[i].previewElement.querySelector('.error').innerHTML = result.description
files[i].previewElement.querySelector('.error').classList.remove('is-hidden')
}
function initUrlUpload (file) {
file.previewElement.querySelector('.descriptive-progress').innerHTML =
'Waiting for server to fetch URL\u2026'

return post(i + 1)
return axios.post('api/upload', {
urls: [file.url]
}, {
headers: {
token: page.token,
albumid: page.album,
age: page.uploadAge,
filelength: page.fileLength
}

files[i].previewElement.querySelector('.descriptive-progress').innerHTML =
'Waiting for server to fetch URL\u2026'

return axios.post('api/upload', { urls: [files[i].url] }, { headers }).then(response => {
return posted(response.data)
}).catch(error => {
return posted({
}).catch(error => {
// Format error for display purpose
return error.response.data ? error.response : {
data: {
success: false,
description: error.response ? error.response.data.description : error.toString()
})
})
description: error.toString()
}
}
}).then(response => {
return finishedUrlUpload(file, response.data)
})
}

function shiftQueue () {
while (page.urlsQueue.length && (page.activeUrlsQueue < page.parallelUploads)) {
page.activeUrlsQueue++
initUrlUpload(page.urlsQueue.shift())
}
return post(0)
}
return run()

return shiftQueue()
}

page.updateTemplateIcon = (templateElement, iconClass) => {
@@ -1,5 +1,5 @@
{
"1": "1574791009",
"1": "1575024012",
"2": "1568894058",
"3": "1568894058",
"4": "1568894058",
@@ -203,9 +203,9 @@
integrity sha512-tHq6qdbT9U1IRSGf14CL0pUlULksvY9OZ+5eEgl1N7t+OA3tGvNpxJCzuKQlsNgCVwbAs670L1vcVQi8j9HjnA==

"@types/node@*":
version "12.12.12"
resolved "https://registry.yarnpkg.com/@types/node/-/node-12.12.12.tgz#529bc3e73dbb35dd9e90b0a1c83606a9d3264bdb"
integrity sha512-MGuvYJrPU0HUwqF7LqvIj50RZUX23Z+m583KBygKYUZLlZ88n6w28XRNJRJgsHukLEnLz6w6SvxZoLgbr5wLqQ==
version "12.12.14"
resolved "https://registry.yarnpkg.com/@types/node/-/node-12.12.14.tgz#1c1d6e3c75dba466e0326948d56e8bd72a1903d2"
integrity sha512-u/SJDyXwuihpwjXy7hOOghagLEV1KdAST6syfnOk6QZAMzZuWZqXy5aYYZbh8Jdpd4escVFP0MvftHNDb9pruA==

"@types/q@^1.5.1":
version "1.5.2"
@@ -646,9 +646,9 @@ aws-sign2@~0.7.0:
integrity sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg=

aws4@^1.8.0:
version "1.8.0"
resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.8.0.tgz#f0e003d9ca9e7f59c7a508945d7b2ef9a04a542f"
integrity sha512-ReZxvNHIOv88FlT7rxcXIIC0fPt4KZqZbOlivyWtXLt8ESx84zd3kMC6iK5jVeS2qt+g7ftS7ye4fi06X5rtRQ==
version "1.9.0"
resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.9.0.tgz#24390e6ad61386b0a747265754d2a17219de862c"
integrity sha512-Uvq6hVe90D0B2WEnUqtdgY1bATGz3mw33nH9Y+dmA+w5DHvUmBgkr5rM/KCHpCsiFNRUfokW/szpPPgMK2hm4A==

bach@^1.0.0:
version "1.2.0"
@@ -721,9 +721,9 @@ bl@^3.0.0:
readable-stream "^3.0.1"

bluebird@^3.7.1:
version "3.7.1"
resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.1.tgz#df70e302b471d7473489acf26a93d63b53f874de"
integrity sha512-DdmyoGCleJnkbp3nkbxTLJ18rjDsE4yCggEwKNXkeV123sPNfOCYeDoeuOY+F2FrSjO1YXcTU+dsy96KMy+gcg==
version "3.7.2"
resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f"
integrity sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==

body-parser@1.19.0, body-parser@^1.19.0:
version "1.19.0"
@@ -2216,9 +2216,9 @@ express@^4.17.1:
vary "~1.1.2"

ext@^1.1.2:
version "1.2.0"
resolved "https://registry.yarnpkg.com/ext/-/ext-1.2.0.tgz#8dd8d2dd21bcced3045be09621fa0cbf73908ba4"
integrity sha512-0ccUQK/9e3NreLFg6K6np8aPyRgwycx+oFGtfx1dSp7Wj00Ozw9r05FgBRlzjf2XBM7LAzwgLyDscRrtSU91hA==
version "1.3.0"
resolved "https://registry.yarnpkg.com/ext/-/ext-1.3.0.tgz#21526eb296753fed34b620d4a69e3911065fa925"
integrity sha512-LErT9cIGZZjSvFkyocVXXeYlj7z8xiA+4oQlM9cX4X/Kfc18cefv3Dd9mNKwFuzUJ7neMMAQz1u1r3gBa/6wGg==
dependencies:
type "^2.0.0"

@@ -3826,10 +3826,10 @@ kind-of@^6.0.0, kind-of@^6.0.2:
resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-6.0.2.tgz#01146b36a6218e64e58f3a8d66de5d7fc6f6d051"
integrity sha512-s5kLOcnH0XqDO+FvuaLX8DDjZ18CGFk7VygH40QoKPUQhW4e2rvM0rwUq0t8IQDOwYSeLK01U90OjzBTme2QqA==

knex@^0.20.2:
version "0.20.2"
resolved "https://registry.yarnpkg.com/knex/-/knex-0.20.2.tgz#7429577a95a10f4a4e3090c23b559fed20343b4a"
integrity sha512-nw7/RsaZrIGdzbsb1evcEaZv8sL/Ji2W7o5OoF0NIKei4ySU01D4G5mRNVNtneoLoPjUMgqSFRanabhGacJUIA==
knex@^0.20.3:
version "0.20.3"
resolved "https://registry.yarnpkg.com/knex/-/knex-0.20.3.tgz#85178cd6873f75827be86d054c4e117bb4d9657b"
integrity sha512-zzYO34pSCCYVqRTbCp8xL+Z7fvHQl5anif3Oacu6JaHFDubB7mFGWRRJBNSO3N8Ql4g4CxUgBctaPiliwoOsNA==
dependencies:
bluebird "^3.7.1"
colorette "1.1.0"
@@ -4414,9 +4414,9 @@ nocache@2.1.0:
integrity sha512-0L9FvHG3nfnnmaEQPjT9xhfN4ISk0A8/2j4M37Np4mcDesJjHgEUfgPhdCyZuFI954tjokaIj/A3NdpFNdEh4Q==

node-abi@^2.7.0:
version "2.12.0"
resolved "https://registry.yarnpkg.com/node-abi/-/node-abi-2.12.0.tgz#40e9cfabdda1837863fa825e7dfa0b15686adf6f"
integrity sha512-VhPBXCIcvmo/5K8HPmnWJyyhvgKxnHTUMXR/XwGHV68+wrgkzST4UmQrY/XszSWA5dtnXpNp528zkcyJ/pzVcw==
version "2.13.0"
resolved "https://registry.yarnpkg.com/node-abi/-/node-abi-2.13.0.tgz#e2f2ec444d0aca3ea1b3874b6de41d1665828f63"
integrity sha512-9HrZGFVTR5SOu3PZAnAY2hLO36aW1wmA+FDsVkr85BTST32TLCA1H/AEcatVRAsWLyXS3bqUDYCAjq5/QGuSTA==
dependencies:
semver "^5.4.1"

@@ -6179,9 +6179,9 @@ resolve-url@^0.2.1:
integrity sha1-LGN/53yJOv0qZj/iGqkIAGjiBSo=

resolve@^1.1.6, resolve@^1.1.7, resolve@^1.10.0, resolve@^1.10.1, resolve@^1.11.0, resolve@^1.3.2, resolve@^1.4.0, resolve@^1.5.0:
version "1.12.2"
resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.12.2.tgz#08b12496d9aa8659c75f534a8f05f0d892fff594"
integrity sha512-cAVTI2VLHWYsGOirfeYVVQ7ZDejtQ9fp4YhYckWDEkFfqbVjaT11iM8k6xSAfGFMM+gDpZjMnFssPu8we+mqFw==
version "1.13.1"
resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.13.1.tgz#be0aa4c06acd53083505abb35f4d66932ab35d16"
integrity sha512-CxqObCX8K8YtAhOBRg+lrcdn+LK+WYOS8tSjqSFbjtrI5PnS63QPhZl4+yKfrU9tdsbMu9Anr/amegT87M9Z6w==
dependencies:
path-parse "^1.0.6"

@@ -6892,10 +6892,10 @@ svgo@^1.0.0:
unquote "~1.1.1"
util.promisify "~1.0.0"

systeminformation@^4.15.3:
version "4.15.3"
resolved "https://registry.yarnpkg.com/systeminformation/-/systeminformation-4.15.3.tgz#639cdc224b5c3811f1a0bfba33f869bbc6fb930f"
integrity sha512-Fx2ARGHtLl2/xLeNoTR8/doXSxUXuAzIN+dyCK9O43j/UETLBt77yTEbTxmYsVD47PYjX1iQTdcY41CZckY+zg==
systeminformation@^4.16.0:
version "4.16.0"
resolved "https://registry.yarnpkg.com/systeminformation/-/systeminformation-4.16.0.tgz#d92ce821efdab720496c60656bf65cc68fb03f8c"
integrity sha512-1FjxPJSw7ad0zug+1YIQATj6Cn+wM5OBASEpjohEeOD2EGPIf0Cnhthd1L2O1YX+wKgOMuPldGfxYdo8yNHEIg==

table@^5.2.3:
version "5.4.6"

0 comments on commit 337a0a6

Please sign in to comment.
You can’t perform that action at this time.