Skip to content
This repository has been archived by the owner on Apr 1, 2024. It is now read-only.

Commit

Permalink
Improved refresh of uploads list + using pending uploads
Browse files Browse the repository at this point in the history 
Signed-off-by: Cédric Foellmi <cedric@onekiloparsec.dev>
  • Loading branch information
@onekiloparsec
onekiloparsec committed May 6, 2020
1 parent 69ca286 commit 6851df0
Show file tree
Hide file tree
Showing 4 changed files with 134 additions and 66 deletions.
37 changes: 28 additions & 9 deletions oort/app/helpers/filesfoldersyncer.py
View file
Expand Up @@ -37,12 +37,24 @@ def walk(self):
filedate = find_xisf_filedate(filepath, self.context.debug)
if filedate is not None:
self.files.append((filepath, filedate))

fu = self.context.uploads.get(filepath)
if fu is None:
fu = FileUploader(filepath,
filedate,
self.astronomer,
self.context.organisation,
self.context.debug,
self.context.verbose)
self.context.uploads[filepath] = fu

else:
# TODO: Deal with missing date
if self.context.debug or self.context.verbose:
print(f'No date found for {filepath}. Skipping.')

if self.context.debug or self.context.verbose:
self._update_upload_lists()
print(f'Found {len(self.files)} files to upload in {self.folderpath}.')

def upload_files(self, telescope_key, resources_key, **raw_resource_kwargs):
Expand Down Expand Up @@ -160,31 +172,38 @@ def _check_existing_remote_resource(self, api: ArcsecondAPI, uuid: str):
return response_detail

def _process_file_upload(self, filepath: str, filedate: datetime, dataset: dict, night_log: dict, telescope: dict):
upload_key = f"dataset_{dataset['uuid']}:{filepath}"
fu = self.context.uploads.get(upload_key)
fu = self.context.uploads.get(filepath)
if fu is None:
fu = FileUploader(filepath,
filedate,
dataset,
night_log,
telescope,
self.astronomer,
self.context.organisation,
self.context.debug,
self.context.verbose)

self.context.uploads[upload_key] = fu
self.context.uploads[filepath] = fu

else:
fu.prepare(dataset, night_log, telescope)

started_count = len([u for u in self.context.uploads.values() if u.is_started()])
if self.context._autostart and started_count < MAX_SIMULTANEOUS_UPLOADS:
fu.start()
if self.context.verbose:
if not fu.is_started() and not fu.is_finished() and self.context.verbose:
print(f'Uploading {filepath}...')
fu.start()

if fu.will_finish():
fu.finish()
if self.context.verbose:
print(f'Finished upload of {filepath}...')

self.context.current_uploads = [fw.to_dict() for fw in self.context.uploads.values() if not fw.is_finished()]
self._update_upload_lists()

def _update_upload_lists(self):
self.context.pending_uploads = [fw.to_dict() for fw in self.context.uploads.values()
if not fw.is_started() and not fw.is_finished()]

self.context.current_uploads = [fw.to_dict() for fw in self.context.uploads.values()
if fw.is_started() and not fw.is_finished()]

self.context.finished_uploads = [fw.to_dict() for fw in self.context.uploads.values() if fw.is_finished()]
74 changes: 42 additions & 32 deletions oort/app/helpers/fileuploader.py
View file
Expand Up @@ -14,9 +14,6 @@ class FileUploader(object):
def __init__(self,
filepath: str,
filedate: datetime,
dataset: dict,
night_log: dict,
telescope: dict,
astronomer: tuple,
organisation: str,
debug: bool,
Expand All @@ -31,25 +28,18 @@ def __init__(self,
if not filedate:
raise ValueError(f'Missing / wrong filedate: {filedate}')

if not dataset or not dataset['uuid']:
raise ValueError(f'Missing / wrong dataset UUID: {dataset["uuid"]}')
try:
uuid.UUID(dataset['uuid'])
except ValueError:
raise ValueError(f'Missing / wrong dataset UUID: {dataset["uuid"]}')

self.filepath = filepath
self.filedate = filedate
self.dataset = dataset
self.night_log = night_log
self.telescope = telescope

self.astronomer = astronomer
self.organisation = organisation

self.debug = debug
self.verbose = verbose

self.dataset = None
self.night_log = None
self.telescope = None

self.filesize = os.path.getsize(filepath)
self.status = 'new'
self.progress = 0
Expand All @@ -60,21 +50,7 @@ def __init__(self,
self.error = None

self._exists_remotely = False

if self.astronomer:
self.api = Arcsecond.build_datafiles_api(dataset=self.dataset['uuid'],
debug=self.debug,
api_key=self.astronomer[1])
else:
self.api = Arcsecond.build_datafiles_api(dataset=self.dataset['uuid'],
debug=self.debug,
organisation=self.organisation)

def update_progress(event, progress_percent):
self.progress = progress_percent
self.duration = (datetime.now() - self.started).total_seconds()

self.uploader, _ = self.api.create({'file': filepath}, callback=update_progress)
self.api = None

@property
def remaining_bytes(self):
Expand Down Expand Up @@ -108,7 +84,37 @@ def exists_remotely(self):
print(f'Multiple files for dataset {self.dataset["uuid"]} and filename {filename}???')
return False

def prepare(self, dataset, night_log, telescope):
if not dataset or not dataset['uuid']:
raise ValueError(f'Missing / wrong dataset UUID: {dataset["uuid"]}')
try:
uuid.UUID(dataset['uuid'])
except ValueError:
raise ValueError(f'Missing / wrong dataset UUID: {dataset["uuid"]}')

self.dataset = dataset
self.night_log = night_log
self.telescope = telescope

if self.astronomer:
self.api = Arcsecond.build_datafiles_api(dataset=self.dataset['uuid'],
debug=self.debug,
api_key=self.astronomer[1])
else:
self.api = Arcsecond.build_datafiles_api(dataset=self.dataset['uuid'],
debug=self.debug,
organisation=self.organisation)

def update_progress(event, progress_percent):
self.progress = progress_percent
self.duration = (datetime.now() - self.started).total_seconds()

self.uploader, _ = self.api.create({'file': self.filepath}, callback=update_progress)

def start(self):
if self.dataset is None:
raise Exception("Missing dataset. Can't start upload.")

if self.started is not None:
return

Expand All @@ -123,9 +129,13 @@ def finish(self):
if self.ended is not None:
return

if not self.exists_remotely():
_, self.error = self.uploader.finish()
if self.exists_remotely():
self.ended = datetime.now()
self.progress = 0
self.duration = (self.ended - self.started).total_seconds()
return

_, self.error = self.uploader.finish()
if self.error:
self.status = 'error'
self._process_error(self.error)
Expand All @@ -134,8 +144,8 @@ def finish(self):
self.status = 'OK'
logger.info(str.ljust('ok', 5) + self.log_string)

self.progress = 0
self.ended = datetime.now()
self.progress = 0
self.duration = (self.ended - self.started).total_seconds()

def _process_error(self, error):
Expand Down
55 changes: 32 additions & 23 deletions oort/app/static/js/uploads.js
View file
Expand Up @@ -8,8 +8,10 @@ var app = new Vue({
selected_telescope: null,
selected_night_log: null,
selected_night_log_url: '',
pending_uploads: [],
current_uploads: [],
finished_uploads: [],
filtered_pending_uploads: [],
filtered_current_uploads: [],
filtered_finished_uploads: [],
source: null,
Expand All @@ -36,9 +38,12 @@ var app = new Vue({
self.selected_night_log_url = 'data/' + self.night_logs[0]['date'].replace(/-/gi, '/')
}

self.pending_uploads = json.pending_uploads
self.current_uploads = json.current_uploads
self.finished_uploads = json.finished_uploads

console.log('onmessage', self.pending_uploads.length, self.current_uploads.length, self.finished_uploads.length)

self.current_uploads.sort((u1, u2) => new Date(u1.started).getDate() < new Date(u2.started).getDate())
self.finished_uploads.sort((u1, u2) => new Date(u1.ended).getDate() < new Date(u2.ended).getDate())

Expand All @@ -50,13 +55,15 @@ var app = new Vue({
}
})

if (self.selected_telescope) {
self.filtered_current_uploads = self.current_uploads.filter(u => u.telescope['uuid'] === self.selected_telescope['uuid'])
self.filtered_finished_uploads = self.finished_uploads.filter(u => u.telescope['uuid'] === self.selected_telescope['uuid'])
} else {
// if (self.selected_telescope) {
// self.filtered_pending_uploads = self.pending_uploads.filter(u => u.telescope['uuid'] === self.selected_telescope['uuid'])
// self.filtered_current_uploads = self.current_uploads.filter(u => u.telescope['uuid'] === self.selected_telescope['uuid'])
// self.filtered_finished_uploads = self.finished_uploads.filter(u => u.telescope['uuid'] === self.selected_telescope['uuid'])
// } else {
self.filtered_pending_uploads = self.pending_uploads
self.filtered_current_uploads = self.current_uploads
self.filtered_finished_uploads = self.finished_uploads
}
// }
}

this.loopID = setInterval(function ping () {
Expand All @@ -68,24 +75,26 @@ var app = new Vue({
},
methods: {
selectTelescope (uuid) {
this.selected_telescope = (uuid === '__all__') ? null : this.telescopes.find(t => t.uuid === uuid)
if (this.selected_telescope) {
this.selected_night_log = this.night_logs.find(nl => nl.telescope === uuid)
this.filtered_current_uploads = this.current_uploads.filter(u => u.telescope['uuid'] === this.selected_telescope['uuid'])
this.filtered_finished_uploads = this.finished_uploads.filter(u => u.telescope['uuid'] === this.selected_telescope['uuid'])
} else {
this.selected_night_log = null
this.filtered_current_uploads = this.current_uploads
this.filtered_finished_uploads = this.finished_uploads
}
if (this.selected_night_log) {
this.selected_night_log_url = 'data/' + this.selected_night_log['date'].replace(/-/gi, '/')
} else if (this.night_logs.length > 0) {
this.selected_night_log_url = 'data/' + this.night_logs[0]['date'].replace(/-/gi, '/')
} else {
this.selected_night_log_url = ''
}
this.$forceUpdate()
// this.selected_telescope = (uuid === '__all__') ? null : this.telescopes.find(t => t.uuid === uuid)
// if (this.selected_telescope) {
// this.selected_night_log = this.night_logs.find(nl => nl.telescope === uuid)
// this.filtered_pending_uploads = this.pending_uploads.filter(u => u.telescope['uuid'] === this.selected_telescope['uuid'])
// this.filtered_current_uploads = this.current_uploads.filter(u => u.telescope['uuid'] === this.selected_telescope['uuid'])
// this.filtered_finished_uploads = this.finished_uploads.filter(u => u.telescope['uuid'] === this.selected_telescope['uuid'])
// } else {
// this.selected_night_log = null
// this.filtered_pending_uploads = this.pending_uploads
// this.filtered_current_uploads = this.current_uploads
// this.filtered_finished_uploads = this.finished_uploads
// }
// if (this.selected_night_log) {
// this.selected_night_log_url = 'data/' + this.selected_night_log['date'].replace(/-/gi, '/')
// } else if (this.night_logs.length > 0) {
// this.selected_night_log_url = 'data/' + this.night_logs[0]['date'].replace(/-/gi, '/')
// } else {
// this.selected_night_log_url = ''
// }
// this.$forceUpdate()
}
}
})
Expand Down
34 changes: 32 additions & 2 deletions oort/app/templates/index.html
View file
Expand Up @@ -267,9 +267,38 @@ <h3>
</div>
</div>

<div class="table-block">
<div class="table-title">Pending Uploads ({{ filtered_pending_uploads.length }})</div>
<table class="table table-striped text-small" v-if="filtered_pending_uploads.length > 0">
<thead>
<tr>
<th>Filename</th>
<th>Owner</th>
<th>Size (kB)</th>
</tr>
</thead>
<tbody>
<tr v-for="upload in filtered_pending_uploads">
<td>
{{ upload.filename }}
<div class="subtitle">{{ upload.filedate }}</div>
</td>
<td>
<a v-if="upload.astronomer" :href=`https://www.arcsecond.io/@${upload.astronomer}` target="_blank">@{{ upload.astronomer }}</a>
<a v-else :href=`https://${upload.organisation}.arcsecond.io` target="_blank">{{ upload.organisation }}</a>
</td>
<td>{{ upload.filesize / 1000 }} kB</td>
</tr>
</tbody>
</table>
<div v-else>
<span class="label-centered">There are no pending uploads.</span>
</div>
</div>

<div class="table-block">
<div class="table-title">Finished Uploads ({{ filtered_finished_uploads.length }})</div>
<table class="table table-striped text-small" v-if="filtered_finished_uploads.length > 0">
<div class="table-block">
<thead>
<tr>
<th>Filename</th>
Expand Down Expand Up @@ -341,10 +370,11 @@ <h3>
</div>
</div>
</div>
</div>

{% endif %}

<script src="{$ url_for('static', filename='js/uploads.js') $}"></script>
<script src="{$ url_for('static', filename='js/uploads.js') $}"></script>

</div>
</body>
Expand Down

0 comments on commit 6851df0

Please sign in to comment.