Skip to content
Permalink
Browse files

Refactor generate and generate_votable

  • Loading branch information...
jochenklar committed Nov 23, 2018
1 parent abf4181 commit 7ea42c115935eb83a8418ae0e6ccf7f576cce783
@@ -6,6 +6,7 @@
from django.conf import settings

from daiquiri.core.generators import generate_csv, generate_votable, generate_fits
from daiquiri.core.utils import get_doi_url

logger = logging.getLogger(__name__)

@@ -16,7 +17,8 @@ def __init__(self, database_key, database_config):
self.database_key = database_key
self.database_config = database_config

def generate(self, format_key, schema_name, table_name, columns, sources=None, status=None, nrows=None):
def generate(self, format_key, columns, sources=[], schema_name=None, table_name=None, nrows=None,
query_status=None, query=None, query_language=None):
# create the final list of arguments subprocess.Popen
if format_key == 'sql':
# create the final list of arguments subprocess.Popen
@@ -35,12 +37,13 @@ def generate(self, format_key, schema_name, table_name, columns, sources=None, s

elif format_key == 'votable':
return generate_votable(self.generate_rows(prepend=prepend), columns,
resource_name=schema_name, table_name=table_name,
sources=sources, query_status=status, empty=(nrows==0))
table_name=self.get_table_name(schema_name, table_name),
infos=self.get_infos(query_status, query, query_language, sources),
links=self.get_links(sources), empty=(nrows==0))

elif format_key == 'fits':
return generate_fits(self.generate_rows(prepend=prepend), columns,
nrows=nrows, table_name=table_name)
return generate_fits(self.generate_rows(prepend=prepend), columns, nrows,
table_name=self.get_table_name(schema_name, table_name))

else:
raise Exception('Not supported.')
@@ -54,8 +57,8 @@ def generate_dump(self):
process = subprocess.Popen(self.args, stdout=subprocess.PIPE)

for line in process.stdout:
if not line.startswith(('\n', '\r\n', '--', 'SET', '/*!')):
yield line
if not line.startswith((b'\n', b'\r\n', b'--', b'SET', b'/*!')):
yield line.decode()

except subprocess.CalledProcessError as e:
logger.error('Command PIPE returned non-zero exit status: %s' % e)
@@ -98,3 +101,31 @@ def get_prepend(self, columns):
prepend[i] = settings.FILES_BASE_URL

return prepend

def get_table_name(self, schema_name, table_name):
return '%(schema_name)s.%(table_name)s' % {
'schema_name': schema_name,
'table_name': table_name
}

def get_infos(self, query_status, query, query_language, sources):
infos = [
{'key': 'QUERY_STATUS', 'value': query_status},
{'key': 'QUERY', 'value': query},
{'key': 'QUERY_LANGUAGE', 'value': query_language},
]

for source in sources:
infos.append({
'key': 'SOURCE',
'value': '%(schema_name)s.%(table_name)s' % source
})

return infos

def get_links(self, sources):
return [{
'title': '%(schema_name)s.%(table_name)s' % source,
'content_role': 'doc',
'href': get_doi_url(source['doi']) if source['doi'] else source['url']
} for source in sources]
@@ -27,39 +27,27 @@ def generate_csv(generator, fields):
yield f.getvalue()


def generate_votable(generator, fields, resource_name=None, table_name=None, sources=None, query_status=None, empty=None):
def generate_votable(generator, fields, infos=[], links=[], table_name=None, empty=None):
yield '''<?xml version="1.0"?>
<VOTABLE version="1.3"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns="http://www.ivoa.net/xml/VOTable/v1.3"
xmlns:stc="http://www.ivoa.net/xml/STC/v1.30">'''

if resource_name:
yield '''
<RESOURCE name="%(name)s" type="results">''' % {'name': resource_name}
else:
yield '''
yield '''
<RESOURCE type="results">'''

if query_status == 'OK':
for info in infos:
yield '''
<INFO name="QUERY_STATUS" value="OK" />'''
elif query_status == 'OVERFLOW':
yield '''
<INFO name="QUERY_STATUS" value="OVERFLOW" />'''
<INFO name="%(key)s" value="%(value)s" />''' % info

if sources:
for source in sources:
yield '''
<LINK name="%(schema_name)s.%(table_name)s" content-role="source" href="%(href)s"/>''' % {
'schema_name': source['schema_name'],
'table_name': source['table_name'],
'href': get_doi_url(source['doi']) or ''
}
for link in links:
yield '''
<LINK title="%(title)s" content-role="%(content_role)s" href="%(href)s"/>''' % link

if table_name:
if table_name is not None:
yield '''
<TABLE name="%(table)s">''' % {'table': table_name}
<TABLE name="%s">''' % table_name
else:
yield '''
<TABLE>'''
@@ -72,7 +60,7 @@ def generate_votable(generator, fields, resource_name=None, table_name=None, sou
attrs.append('%s="%s"' % (key, value))

if 'arraysize' in field and field['arraysize']:
attrs.append('arraysize="%d"' % field['arraysize'])
attrs.append('arraysize="%s"' % field['arraysize'])

if field['datatype'] in ['char', 'unsignedByte', 'short', 'int', 'long', 'float', 'double']:
attrs.append('datatype="%s"' % field['datatype'])
@@ -65,6 +65,7 @@
MEETINGS_ABSTRACT_MAX_LENGTH = 2000

METADATA_COLUMN_PERMISSIONS = False
METADATA_BASE_URL = None

QUERY_ANONYMOUS = False
QUERY_USER_SCHEMA_PREFIX = 'daiquiri_user_'
@@ -26,13 +26,13 @@ class FileTests(TestViewMixin, FilesViewTestCase):

status_map = {
'html': {
'admin': 200, 'manager': 200, 'user': 200, 'anonymous': 404
'admin': 200, 'manager': 200, 'user': 200, 'anonymous': 302
},
'html_a': {
'admin': 200, 'manager': 200, 'user': 200, 'anonymous': 404
'admin': 200, 'manager': 200, 'user': 200, 'anonymous': 302
},
'html_a_a': {
'admin': 404, 'manager': 200, 'user': 404, 'anonymous': 404
'admin': 403, 'manager': 200, 'user': 403, 'anonymous': 302
},
'html_a_b': {
'admin': 404, 'manager': 404, 'user': 404, 'anonymous': 404
@@ -16,7 +16,6 @@
from jsonfield import JSONField

from daiquiri.core.adapter import DatabaseAdapter, DownloadAdapter
from daiquiri.core.generators import generate_votable
from daiquiri.core.constants import ACCESS_LEVEL_CHOICES
from daiquiri.jobs.models import Job
from daiquiri.jobs.managers import JobManager
@@ -258,12 +257,15 @@ def run_sync(self):
)

try:
return generate_votable(
adapter.fetchall(self.actual_query),
return DownloadAdapter().generate(
'votable',
get_job_columns(self),
table_name=self.table_name,
sources=job_sources,
query_status='OK'
schema_name=self.schema_name,
table_name=self.table_name,
query_status='OK',
query=self.query,
query_language=self.query_language
)
except (OperationalError, ProgrammingError, InternalError, DataError) as e:
self.error_summary = str(e)
@@ -325,12 +327,14 @@ def stream(self, format_key):
if self.phase == self.PHASE_COMPLETED:
return DownloadAdapter().generate(
format_key,
self.schema_name,
self.table_name,
self.metadata.get('columns'),
self.metadata.get('sources'),
self.result_status,
self.nrows
self.metadata.get('columns', []),
sources=self.metadata.get('sources', []),
schema_name=self.schema_name,
table_name=self.table_name,
nrows=self.nrows,
query_status=self.result_status,
query=self.query,
query_language=self.query_language
)
else:
raise ValidationError({
@@ -256,13 +256,18 @@ def get_job_sources(job):
schema__name=schema_name
)

if settings.METADATA_BASE_URL:
metadata_url = '%s/%s/%s' % (settings.METADATA_BASE_URL.strip('/'), schema_name, table_name)
else:
metadata_url = ''

table.update({
'title': original_table.title,
'description': original_table.description,
'attribution': original_table.attribution,
'license': original_table.license,
'doi': original_table.doi,
'url': reverse('metadata:table', args=[schema_name, table_name])
'url': metadata_url
})

sources.append(table)
@@ -313,14 +313,13 @@ def stream(self, request, pk=None, format_key=None):
# check if the file was lost
if download_job.phase == download_job.PHASE_COMPLETED and os.path.isfile(download_job.file_path):
# stream the previously created file
return sendfile(request, download_job.file_path, attachment=True)
return sendfile(request, download_job.file_path)
except DownloadJob.DoesNotExist:
pass

# stream the table directly from the database
file_name = '%s.%s' % (job.table_name, format_config['extension'])
response = FileResponse(job.stream(format_key), content_type=format_config['content_type'])
response['Content-Disposition'] = "attachment; filename=%s" % file_name
return response


@@ -188,7 +188,7 @@
"order": 2,
"name": "ra",
"description": "Right ascension",
"unit": "Angle[deg]",
"unit": "deg",
"ucd": "pos.eq.ra;meta.main",
"utype": null,
"datatype": "double",
@@ -209,7 +209,7 @@
"order": 3,
"name": "dec",
"description": "Declination",
"unit": "Angle[deg]",
"unit": "deg",
"ucd": "pos.eq.dec;meta.main",
"utype": null,
"datatype": "double",
@@ -230,7 +230,7 @@
"order": 4,
"name": "parallax",
"description": "Parallax",
"unit": "Angle[mas]",
"unit": "mas",
"ucd": "pos.parallax",
"utype": null,
"datatype": "double",
@@ -618,7 +618,7 @@
"order": 6,
"name": "ra",
"description": "Right ascension",
"unit": "Angle[deg]",
"unit": "deg",
"ucd": "pos.eq.ra;meta.main",
"utype": null,
"datatype": "double",
@@ -641,7 +641,7 @@
"order": 7,
"name": "dec",
"description": "Declination",
"unit": "Angle[deg]",
"unit": "deg",
"ucd": "pos.eq.dec;meta.main",
"utype": null,
"datatype": "double",

0 comments on commit 7ea42c1

Please sign in to comment.
You can’t perform that action at this time.