From 367444e13ed3ffc852490a0c732a22e823d0064a Mon Sep 17 00:00:00 2001 From: Andres Aguilera Date: Thu, 9 Jan 2020 09:41:49 -0300 Subject: [PATCH 01/26] Add output to google drive option --- default/google_options.py | 35 +++++++++++++++++++++++++++++++++++ default/views.py | 1 + 2 files changed, 36 insertions(+) create mode 100644 default/google_options.py diff --git a/default/google_options.py b/default/google_options.py new file mode 100644 index 0000000..54e08b8 --- /dev/null +++ b/default/google_options.py @@ -0,0 +1,35 @@ +from __future__ import print_function +from googleapiclient.discovery import build +from google_auth_oauthlib.flow import InstalledAppFlow +from google.auth.transport.requests import Request +from oauth2client import file, client, tools +from apiclient.http import MediaFileUpload, MediaIoBaseDownload + + +def upload_option(file_name, file_path): + creds = None + # Setup the Drive v3 API + SCOPES = 'https://www.googleapis.com/auth/drive.file' + if not creds or not creds.valid: + if creds and creds.expired and creds.refresh_token: + creds.refresh(Request()) + else: + flow = InstalledAppFlow.from_client_secrets_file( + 'credentials.json', SCOPES) + creds = flow.run_local_server(port=0) + if not creds or not creds.valid: + flow = client.flow_from_clientsecrets('client_secret.json', SCOPES) + creds = tools.run_flow(flow, store) + service = build('drive', 'v3', credentials=creds) + + file_metadata = { + 'name': file_name, + 'mimeType': '*/*' + } + media = MediaFileUpload(file_path, + mimetype='*/*', + resumable=True) + file_upload = service.files().create(body=file_metadata, media_body=media, fields='id').execute() + id_file = file_upload.get('id') + + return id_file diff --git a/default/views.py b/default/views.py index 23f7f83..e39b69d 100644 --- a/default/views.py +++ b/default/views.py @@ -17,6 +17,7 @@ from default.data_file import DataFile from default.decorators import clear_files, published_date, require_files from default.forms import MappingSheetOptionsForm +from default.google_options import upload_option from default.mapping_sheet import (get_extended_mapping_sheet, get_mapping_sheet_from_uploaded_file, get_mapping_sheet_from_url) from default.util import (get_files_from_session, invalid_request_file_message, json_response, make_package, From 630130bf48342d8d038898bcfec4417e4dd4e9ee Mon Sep 17 00:00:00 2001 From: Andres Aguilera Date: Fri, 10 Jan 2020 13:35:52 -0300 Subject: [PATCH 02/26] Add upload to google drive options --- default/drive_options.py | 46 +++++++++++++++++++ default/google_options.py | 35 -------------- default/static/js/uploader.js | 1 + default/templates/default/base-uploader.html | 7 ++- default/templates/default/to-json.html | 5 +- default/templates/default/to-spreadsheet.html | 10 +++- default/urls.py | 3 ++ default/views.py | 16 +++++-- ocdstoucan/settings.py | 2 + 9 files changed, 82 insertions(+), 43 deletions(-) create mode 100644 default/drive_options.py delete mode 100644 default/google_options.py diff --git a/default/drive_options.py b/default/drive_options.py new file mode 100644 index 0000000..40b236c --- /dev/null +++ b/default/drive_options.py @@ -0,0 +1,46 @@ +from __future__ import print_function +from django.http import HttpResponse +from googleapiclient.discovery import build +from google_auth_oauthlib.flow import InstalledAppFlow +from google.auth.transport.requests import Request +from googleapiclient.errors import UnknownFileType +from googleapiclient.http import MediaFileUpload +from oauthlib.oauth2 import AccessDeniedError +from ocdstoucan.settings import OCDS_TOUCAN_CREDENTIALS_DRIVE + + +def upload_to_drive(filename, filepath, format=None): + try: + credentials = None + SCOPES = 'https://www.googleapis.com/auth/drive.file' + if not credentials or not credentials.valid: + if credentials and credentials.expired and credentials.refresh_token: + credentials.refresh(Request()) + else: + flow = InstalledAppFlow.from_client_secrets_file( + OCDS_TOUCAN_CREDENTIALS_DRIVE, SCOPES) + credentials = flow.run_local_server(port=0) + service = build('drive', 'v3', credentials=credentials) + + except AccessDeniedError: + return HttpResponse('Access Denied to Google Drive', status=400) + + try: + if format == 'xlsx': + mimeType = 'application/vnd.google-apps.spreadsheet' + else: + mimeType = '*/*' + + file_metadata = { + 'name': filename, + 'mimeType': mimeType + } + media = MediaFileUpload(filepath, + mimetype=mimeType, + resumable=True) + service.files().create(body=file_metadata, media_body=media, fields='id').execute() + + return HttpResponse('Uploaded to Google Drive: Check your account', status=200) + + except (TypeError, Exception, IOError, UnknownFileType): + return HttpResponse('Fail uploading to Google Drive', status=400) diff --git a/default/google_options.py b/default/google_options.py deleted file mode 100644 index 54e08b8..0000000 --- a/default/google_options.py +++ /dev/null @@ -1,35 +0,0 @@ -from __future__ import print_function -from googleapiclient.discovery import build -from google_auth_oauthlib.flow import InstalledAppFlow -from google.auth.transport.requests import Request -from oauth2client import file, client, tools -from apiclient.http import MediaFileUpload, MediaIoBaseDownload - - -def upload_option(file_name, file_path): - creds = None - # Setup the Drive v3 API - SCOPES = 'https://www.googleapis.com/auth/drive.file' - if not creds or not creds.valid: - if creds and creds.expired and creds.refresh_token: - creds.refresh(Request()) - else: - flow = InstalledAppFlow.from_client_secrets_file( - 'credentials.json', SCOPES) - creds = flow.run_local_server(port=0) - if not creds or not creds.valid: - flow = client.flow_from_clientsecrets('client_secret.json', SCOPES) - creds = tools.run_flow(flow, store) - service = build('drive', 'v3', credentials=creds) - - file_metadata = { - 'name': file_name, - 'mimeType': '*/*' - } - media = MediaFileUpload(file_path, - mimetype='*/*', - resumable=True) - file_upload = service.files().create(body=file_metadata, media_body=media, fields='id').execute() - id_file = file_upload.get('id') - - return id_file diff --git a/default/static/js/uploader.js b/default/static/js/uploader.js index 35c0792..4cbb116 100644 --- a/default/static/js/uploader.js +++ b/default/static/js/uploader.js @@ -127,6 +127,7 @@ var app = {}; .done(function (data) { $('.response-success .file-size').html(utils.readableFileSize(data.size)); $('.response-success .download').attr('href', data.url); + $('.response-success .download-drive').attr('href', data.url + 'drive/'); $('.response-success').removeClass('hidden'); if (data.hasOwnProperty('warnings') && data.warnings.length > 0) { $('.response-warning.action-failed').removeClass('hidden'); diff --git a/default/templates/default/base-uploader.html b/default/templates/default/base-uploader.html index 8fc39c1..d161d16 100644 --- a/default/templates/default/base-uploader.html +++ b/default/templates/default/base-uploader.html @@ -12,8 +12,11 @@ {% block body %} @@ -112,6 +114,7 @@ .done(function(data){ $('.response-success .download-json').attr('href', data.url); $('.response-success .file-size-json').html(utils.readableFileSize(data.size)); + $('.response-success .download-drive-json').attr('href', data.url + 'drive/'); $('.response-success').removeClass('hidden'); }) .fail(function(){ diff --git a/default/templates/default/to-spreadsheet.html b/default/templates/default/to-spreadsheet.html index 8d2154b..832ff44 100644 --- a/default/templates/default/to-spreadsheet.html +++ b/default/templates/default/to-spreadsheet.html @@ -16,11 +16,15 @@ @@ -110,6 +114,8 @@ $('.response-success .download-csv').attr('href', data.csv.url); $('.response-success .file-size-xlsx').html(utils.readableFileSize(data.xlsx.size)); $('.response-success .file-size-csv').html(utils.readableFileSize(data.csv.size)); + $('.response-success .download-drive-xlsx').attr('href', data.xlsx.url + 'drive/'); + $('.response-success .download-drive-csv').attr('href', data.csv.url + 'drive/'); $('.response-success').removeClass('hidden'); }) .fail(function(){ diff --git a/default/urls.py b/default/urls.py index f39ffe7..4348f73 100644 --- a/default/urls.py +++ b/default/urls.py @@ -7,6 +7,9 @@ path('i18n/', include('django.conf.urls.i18n')), path('upload/', views.uploadfile, name='upload'), path('delete/', views.deletefile, name='delete_file'), + path('result///drive/', views.retrieve_result_drive, name='retrieve_result_drive'), + path('result////drive/', views.retrieve_result_drive, + name='retrieve_result_drive'), path('result///', views.retrieve_result, name='retrieve_result'), path('result////', views.retrieve_result, name='retrieve_result'), path('upgrade/', views.upgrade, name='upgrade'), diff --git a/default/views.py b/default/views.py index 83c38f0..eba9b0f 100644 --- a/default/views.py +++ b/default/views.py @@ -17,14 +17,14 @@ from default.data_file import DataFile from default.decorators import clear_files, published_date, require_files from default.forms import MappingSheetOptionsForm -from default.google_options import upload_option +from default.drive_options import upload_to_drive from default.mapping_sheet import (get_extended_mapping_sheet, get_mapping_sheet_from_uploaded_file, get_mapping_sheet_from_url) from default.util import (get_files_from_session, invalid_request_file_message, json_response, make_package, ocds_command) -def retrieve_result(request, folder, id, format=None): +def retrieve_result(request, folder, id, format=None, output=None): if format is None: prefix = 'result' ext = '.zip' @@ -41,7 +41,17 @@ def retrieve_result(request, folder, id, format=None): raise Http404('Invalid option') file = DataFile(prefix, ext, id=str(id), folder=folder) - return FileResponse(open(file.path, 'rb'), filename=filename, as_attachment=True) + + if output is None: + return FileResponse(open(file.path, 'rb'), filename=filename, as_attachment=True) + elif output == 'drive': + return upload_to_drive(filename, file.path, format) + else: + raise Http404('Invalid output') + + +def retrieve_result_drive(request, folder, id, format=None): + return retrieve_result(request, folder, id, format, output='drive') def index(request): diff --git a/ocdstoucan/settings.py b/ocdstoucan/settings.py index 3aa5ae6..6947837 100644 --- a/ocdstoucan/settings.py +++ b/ocdstoucan/settings.py @@ -147,3 +147,5 @@ dsn=os.getenv('SENTRY_DSN'), integrations=[DjangoIntegration()] ) + +OCDS_TOUCAN_CREDENTIALS_DRIVE = os.getenv('OCDS_TOUCAN_CREDENTIALS_DRIVE', 'credentials.json') From 7a54a7f5f1fbe12dd5900a8d87def89f3a089940 Mon Sep 17 00:00:00 2001 From: Andres Aguilera Date: Fri, 10 Jan 2020 13:39:32 -0300 Subject: [PATCH 03/26] Add translations --- locale/es/LC_MESSAGES/django.po | 100 +++++++++++++++++--------------- 1 file changed, 54 insertions(+), 46 deletions(-) diff --git a/locale/es/LC_MESSAGES/django.po b/locale/es/LC_MESSAGES/django.po index 52889b6..1cf5644 100644 --- a/locale/es/LC_MESSAGES/django.po +++ b/locale/es/LC_MESSAGES/django.po @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: PACKAGE VERSION\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2019-12-19 18:00-0300\n" +"POT-Creation-Date: 2020-01-10 13:35-0300\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" @@ -76,37 +76,44 @@ msgstr "Indique al menos una URL de extensión" msgid "Success!" msgstr "Éxito!" -#: default/templates/default/base-uploader.html:16 +#: default/templates/default/base-uploader.html:17 #: default/templates/default/to-json.html:19 #: default/templates/default/to-spreadsheet.html:19 -#: default/templates/default/to-spreadsheet.html:23 -msgid "Download" -msgstr "Descargar" - -#: default/templates/default/base-uploader.html:22 +#: default/templates/default/to-spreadsheet.html:25 +msgid "Local download" +msgstr "Descarga local" + +#: default/templates/default/base-uploader.html:19 +#: default/templates/default/to-json.html:21 +#: default/templates/default/to-spreadsheet.html:21 +#: default/templates/default/to-spreadsheet.html:27 +msgid "Upload to Google Drive" +msgstr "Subir a Google Drive" + +#: default/templates/default/base-uploader.html:25 msgid "" "We couldn't check that all files were JSON or of the OCDS type requested." msgstr "" "No pudimos comprobar que todos los archivos sean JSON, o del tipo OCDS " "solicitado" -#: default/templates/default/base-uploader.html:23 +#: default/templates/default/base-uploader.html:26 msgid "" "You can add more files or remove them, then click on Start again." msgstr "" "Puede agregar más archivos o eliminarlos, luego puede hacer clic en " "Procesar otra vez" -#: default/templates/default/base-uploader.html:24 +#: default/templates/default/base-uploader.html:27 msgid "Files with validation issues will be ignored by default." msgstr "Los archivos con problemas de validación serán ignorados por defecto." -#: default/templates/default/base-uploader.html:27 -#: default/templates/default/to-json.html:24 +#: default/templates/default/base-uploader.html:30 +#: default/templates/default/to-json.html:26 msgid "An error has occurred!" msgstr "¡Ha ocurrido un error!" -#: default/templates/default/base-uploader.html:28 +#: default/templates/default/base-uploader.html:31 msgid "" "Please verify that all your files are valid OCDS JSON, and try again in a " "few minutes." @@ -114,31 +121,31 @@ msgstr "" "Por favor verifique que todos sus archivos son válidos de acuerdo al " "estándar OCDS, e intente de nuevo en unos minutos." -#: default/templates/default/base-uploader.html:53 -#: default/templates/default/to-json.html:58 -#: default/templates/default/to-spreadsheet.html:54 +#: default/templates/default/base-uploader.html:56 +#: default/templates/default/to-json.html:60 +#: default/templates/default/to-spreadsheet.html:58 msgid "Add a file" msgstr "Agregue un archivo" -#: default/templates/default/base-uploader.html:53 -#: default/templates/default/to-json.html:59 -#: default/templates/default/to-spreadsheet.html:55 +#: default/templates/default/base-uploader.html:56 +#: default/templates/default/to-json.html:61 +#: default/templates/default/to-spreadsheet.html:59 msgid "or drag and drop here." msgstr "o arrastre y suelte aquí" -#: default/templates/default/base-uploader.html:61 +#: default/templates/default/base-uploader.html:64 msgid "Add more files" msgstr "Agregar más archivos" -#: default/templates/default/base-uploader.html:65 -#: default/templates/default/to-json.html:65 -#: default/templates/default/to-spreadsheet.html:61 +#: default/templates/default/base-uploader.html:68 +#: default/templates/default/to-json.html:67 +#: default/templates/default/to-spreadsheet.html:65 msgid "Start" msgstr "Procesar" -#: default/templates/default/base-uploader.html:75 -#: default/templates/default/to-json.html:73 -#: default/templates/default/to-spreadsheet.html:69 +#: default/templates/default/base-uploader.html:78 +#: default/templates/default/to-json.html:75 +#: default/templates/default/to-spreadsheet.html:73 msgid "" "Processing operation. Please don't refresh nor close the browser window!" msgstr "" @@ -375,7 +382,7 @@ msgstr "" msgid "Published date:" msgstr "Fecha de Publicación:" -#: default/templates/default/to-json.html:25 +#: default/templates/default/to-json.html:27 msgid "" "Please verify that all your files are valid according to OCDS, and try again " "in a few minutes." @@ -383,7 +390,7 @@ msgstr "" "Por favor verifique que todos sus archivos son válidos de acuerdo al " "estándar OCDS, e intente de nuevo en unos minutos." -#: default/templates/default/to-json.html:30 +#: default/templates/default/to-json.html:32 msgid "" "Use this page to convert a CSV or Excel version into a Agregue un archivo." -#: default/templates/default/to-json.html:36 +#: default/templates/default/to-json.html:38 msgid "" "The input CSV files must be compressed at the root of a ZIP file and the " "Excel file must be an XLSX file. Please verify that your files are in a " @@ -409,12 +416,12 @@ msgstr "" "flatten-tool.readthedocs.io/en/latest/usage-ocds/\" target=\"_blank" "\">Flatten Tool para OCDS." -#: default/templates/default/to-json.html:95 -#: default/templates/default/to-spreadsheet.html:91 +#: default/templates/default/to-json.html:97 +#: default/templates/default/to-spreadsheet.html:95 msgid "Added:" msgstr "Añadido:" -#: default/templates/default/to-spreadsheet.html:32 +#: default/templates/default/to-spreadsheet.html:36 msgid "" "Use this page to convert a release package to " @@ -427,23 +434,24 @@ msgstr "" "un archivo arrastrándolo y soltándolo en la caja de abajo o usando el enlace " "Añadir un archivo." -#: default/templates/default/to-spreadsheet.html:117 +#: default/templates/default/to-spreadsheet.html:123 msgid "" "An error has occurred! Please verify that all your files are valid OCDS " "JSON, and try again in a few minutes." msgstr "" -"¡Ha ocurrido un error! Por favor verifique que todos sus archivos son válidos de acuerdo al " -"estándar OCDS, e intente de nuevo en unos minutos." +"¡Ha ocurrido un error! Por favor verifique que todos sus archivos son " +"válidos de acuerdo al estándar OCDS, e intente de nuevo en unos minutos." -#: default/templates/default/to-spreadsheet.html:127 +#: default/templates/default/to-spreadsheet.html:133 msgid "Error when processing file: " msgstr "Error al procesar el archivo:" -#: default/templates/default/to-spreadsheet.html:130 +#: default/templates/default/to-spreadsheet.html:136 msgid "" "Please verify that the file is a valid OCDS Release Package and try again." msgstr "" -"Favor verifique que su archivo es un Paquete de Release OCDS válido e intente de nuevo." +"Favor verifique que su archivo es un Paquete de Release OCDS válido e " +"intente de nuevo." #: default/templates/default/upgrade.html:10 msgid "" @@ -475,40 +483,40 @@ msgstr "Borrar" msgid "Processing..." msgstr "Procesando..." -#: default/util.py:68 +#: default/util.py:65 msgid "Not a record package" msgstr "No es un paquete de records" -#: default/util.py:71 +#: default/util.py:68 msgid "Not a release package" msgstr "No es un paquete de releases" -#: default/util.py:74 +#: default/util.py:71 msgid "Not a release or package" msgstr "No es un release o paquete" -#: default/util.py:78 +#: default/util.py:75 msgid "Not a package or list of packages" msgstr "No es un paquete o lista de paquetes" -#: default/util.py:82 +#: default/util.py:79 msgid "Not a release or list of releases" msgstr "No es un release o lista de releases" -#: default/util.py:84 +#: default/util.py:81 #, python-format msgid "\"%(type)s\" not recognized" msgstr "No se reconoce \"%(type)s\"" -#: default/util.py:86 +#: default/util.py:83 msgid "Error decoding JSON" msgstr "Error al decodificar JSON" -#: default/views.py:279 default/views.py:287 +#: default/views.py:290 default/views.py:298 msgid "File not found" msgstr "Archivo no encontrado" -#: default/views.py:285 +#: default/views.py:296 msgid "File deleted" msgstr "Archivo eliminado" From 3f0d73433acaf29843409f1389c1cc4e3f830b92 Mon Sep 17 00:00:00 2001 From: Andres Aguilera Date: Fri, 10 Jan 2020 14:07:23 -0300 Subject: [PATCH 04/26] Fix imports --- default/drive_options.py | 4 ++-- default/views.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/default/drive_options.py b/default/drive_options.py index 40b236c..a52429e 100644 --- a/default/drive_options.py +++ b/default/drive_options.py @@ -1,10 +1,10 @@ from __future__ import print_function from django.http import HttpResponse -from googleapiclient.discovery import build from google_auth_oauthlib.flow import InstalledAppFlow -from google.auth.transport.requests import Request +from googleapiclient.discovery import build from googleapiclient.errors import UnknownFileType from googleapiclient.http import MediaFileUpload +from google.auth.transport.requests import Request from oauthlib.oauth2 import AccessDeniedError from ocdstoucan.settings import OCDS_TOUCAN_CREDENTIALS_DRIVE diff --git a/default/views.py b/default/views.py index eba9b0f..0287492 100644 --- a/default/views.py +++ b/default/views.py @@ -16,8 +16,8 @@ from default.data_file import DataFile from default.decorators import clear_files, published_date, require_files -from default.forms import MappingSheetOptionsForm from default.drive_options import upload_to_drive +from default.forms import MappingSheetOptionsForm from default.mapping_sheet import (get_extended_mapping_sheet, get_mapping_sheet_from_uploaded_file, get_mapping_sheet_from_url) from default.util import (get_files_from_session, invalid_request_file_message, json_response, make_package, From ebe8ea1cd5235e570c88f0992df469ffe24101b4 Mon Sep 17 00:00:00 2001 From: Andres Aguilera Date: Fri, 10 Jan 2020 14:10:22 -0300 Subject: [PATCH 05/26] Fix warning --- default/drive_options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/default/drive_options.py b/default/drive_options.py index a52429e..06764de 100644 --- a/default/drive_options.py +++ b/default/drive_options.py @@ -1,10 +1,10 @@ from __future__ import print_function from django.http import HttpResponse +from google.auth.transport.requests import Request from google_auth_oauthlib.flow import InstalledAppFlow from googleapiclient.discovery import build from googleapiclient.errors import UnknownFileType from googleapiclient.http import MediaFileUpload -from google.auth.transport.requests import Request from oauthlib.oauth2 import AccessDeniedError from ocdstoucan.settings import OCDS_TOUCAN_CREDENTIALS_DRIVE From 930143eb7f22f9fbf9382be1861a892667a3a5ba Mon Sep 17 00:00:00 2001 From: Andres Aguilera Date: Fri, 24 Jan 2020 09:33:21 -0300 Subject: [PATCH 06/26] Add translations --- default/drive_options.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/default/drive_options.py b/default/drive_options.py index 06764de..47a7308 100644 --- a/default/drive_options.py +++ b/default/drive_options.py @@ -23,7 +23,7 @@ def upload_to_drive(filename, filepath, format=None): service = build('drive', 'v3', credentials=credentials) except AccessDeniedError: - return HttpResponse('Access Denied to Google Drive', status=400) + return HttpResponse('Access Denied
Acceso Denegado', status=400) try: if format == 'xlsx': @@ -40,7 +40,7 @@ def upload_to_drive(filename, filepath, format=None): resumable=True) service.files().create(body=file_metadata, media_body=media, fields='id').execute() - return HttpResponse('Uploaded to Google Drive: Check your account', status=200) + return HttpResponse('Uploaded to Google Drive: Check your account
Subido a Google Drive: Verifica tu cuenta', status=200) except (TypeError, Exception, IOError, UnknownFileType): - return HttpResponse('Fail uploading to Google Drive', status=400) + return HttpResponse('Fail uploading to Google Drive
Fallo al subir a Google Drive', status=400) From 5cb154920616732ed4a46151d86c8704b83c771b Mon Sep 17 00:00:00 2001 From: Andres Aguilera Date: Fri, 24 Jan 2020 10:46:53 -0300 Subject: [PATCH 07/26] Add module google --- requirements.in | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/requirements.in b/requirements.in index 8ca7be4..901ddf1 100644 --- a/requirements.in +++ b/requirements.in @@ -1,10 +1,16 @@ Django>=2.1.11,<2.2 flattentool +google +google-api-python-client +google-auth +google-auth-httplib2 +google-auth-oauthlib jsonref libcoveocds ocdskit[perf] ocdsmerge ocdsextensionregistry +oauthlib python-dateutil requests sentry-sdk From 7cb11ca018b231068fecc8aa3f73e5e38d81b14c Mon Sep 17 00:00:00 2001 From: Andres Aguilera Date: Fri, 24 Jan 2020 10:59:32 -0300 Subject: [PATCH 08/26] Fix large line --- default/drive_options.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/default/drive_options.py b/default/drive_options.py index 47a7308..c138269 100644 --- a/default/drive_options.py +++ b/default/drive_options.py @@ -40,7 +40,10 @@ def upload_to_drive(filename, filepath, format=None): resumable=True) service.files().create(body=file_metadata, media_body=media, fields='id').execute() - return HttpResponse('Uploaded to Google Drive: Check your account
Subido a Google Drive: Verifica tu cuenta', status=200) + return HttpResponse( + 'Uploaded to Google Drive: Check your account
Subido a Google Drive: Verifica tu cuenta', + status=200 + ) except (TypeError, Exception, IOError, UnknownFileType): return HttpResponse('Fail uploading to Google Drive
Fallo al subir a Google Drive', status=400) From 416c418abf39d8e02d770de821797f2e5b5d5305 Mon Sep 17 00:00:00 2001 From: Andres Aguilera Date: Fri, 31 Jan 2020 17:39:39 -0300 Subject: [PATCH 09/26] Update requeriments --- requirements.txt | 17 ++++++++++++++++- requirements_dev.txt | 15 +++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 121f2d7..32a89ab 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,8 +5,10 @@ # pip-compile # attrs==19.3.0 # via jsonschema +beautifulsoup4==4.8.2 # via google bleach==3.1.0 # via libcove, libcoveocds cached-property==1.5.1 # via libcove, libcoveocds +cachetools==4.0.0 # via google-auth certifi==2019.9.11 # via requests, sentry-sdk chardet==3.0.4 # via requests commonmark==0.9.1 # via libcove, libcoveocds @@ -14,6 +16,12 @@ contextlib2==0.5.5 # via schema django==2.1.15 et-xmlfile==1.0.1 # via openpyxl flattentool==0.9.0 +google-api-python-client==1.7.11 +google-auth-httplib2==0.0.3 +google-auth-oauthlib==0.4.1 +google-auth==1.11.0 +google==2.0.3 +httplib2==0.17.0 # via google-api-python-client, google-auth-httplib2 idna==2.8 # via requests ijson==2.5.1 # via ocdskit importlib-metadata==1.3.0 # via jsonschema @@ -25,22 +33,29 @@ libcove==0.12.0 # via libcoveocds libcoveocds==0.7.4 lxml==4.4.1 # via flattentool more-itertools==8.0.2 # via zipp +oauthlib==3.1.0 ocdsextensionregistry==0.0.16 ocdskit[perf]==0.2.1 ocdsmerge==0.6.4 openpyxl==3.0.1 # via flattentool orjson==2.1.3 # via ocdskit +pyasn1-modules==0.2.8 # via google-auth +pyasn1==0.4.8 # via pyasn1-modules, rsa pyrsistent==0.15.5 # via jsonschema python-dateutil==2.8.1 pytz==2019.3 # via django, flattentool requests-cache==0.5.2 # via ocdsextensionregistry +requests-oauthlib==1.3.0 # via google-auth-oauthlib requests==2.22.0 rfc3987==1.3.8 # via ocdskit +rsa==4.0 # via google-auth schema==0.7.1 # via flattentool sentry-sdk==0.13.2 -six==1.13.0 # via bleach, flattentool, jsonschema, pyrsistent, python-dateutil +six==1.13.0 # via bleach, flattentool, google-api-python-client, google-auth, jsonschema, pyrsistent, python-dateutil +soupsieve==1.9.5 # via beautifulsoup4 sqlalchemy==1.3.11 # via ocdskit strict-rfc3339==0.7 # via ocdskit +uritemplate==3.0.1 # via google-api-python-client urllib3==1.25.7 # via requests, sentry-sdk webencodings==0.5.1 # via bleach xmltodict==0.12.0 # via flattentool diff --git a/requirements_dev.txt b/requirements_dev.txt index 21eb308..af9caea 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -5,8 +5,10 @@ # pip-compile requirements_dev.in # attrs==19.3.0 +beautifulsoup4==4.8.2 bleach==3.1.0 cached-property==1.5.1 +cachetools==4.0.0 certifi==2019.9.11 chardet==3.0.4 click==7.0 # via pip-tools @@ -18,6 +20,12 @@ entrypoints==0.3 # via flake8 et-xmlfile==1.0.1 flake8==3.7.9 flattentool==0.9.0 +google-api-python-client==1.7.11 +google-auth-httplib2==0.0.3 +google-auth-oauthlib==0.4.1 +google-auth==1.11.0 +google==2.0.3 +httplib2==0.17.0 idna==2.8 ijson==2.5.1 importlib-metadata==1.3.0 @@ -31,25 +39,32 @@ libcoveocds==0.7.4 lxml==4.4.1 mccabe==0.6.1 # via flake8 more-itertools==8.0.2 +oauthlib==3.1.0 ocdsextensionregistry==0.0.16 ocdskit[perf]==0.2.1 ocdsmerge==0.6.4 openpyxl==3.0.1 orjson==2.1.3 pip-tools==4.3.0 +pyasn1-modules==0.2.8 +pyasn1==0.4.8 pycodestyle==2.5.0 # via flake8 pyflakes==2.1.1 # via flake8 pyrsistent==0.15.5 python-dateutil==2.8.1 pytz==2019.3 requests-cache==0.5.2 +requests-oauthlib==1.3.0 requests==2.22.0 rfc3987==1.3.8 +rsa==4.0 schema==0.7.1 sentry-sdk==0.13.2 six==1.13.0 +soupsieve==1.9.5 sqlalchemy==1.3.11 strict-rfc3339==0.7 +uritemplate==3.0.1 urllib3==1.25.7 webencodings==0.5.1 xmltodict==0.12.0 From 27066d4b033006fd291735c45087f01c6cc2331a Mon Sep 17 00:00:00 2001 From: Andres Aguilera Date: Thu, 13 Feb 2020 08:48:41 -0300 Subject: [PATCH 10/26] Update translations --- default/templates/default/base-uploader.html | 4 +- default/templates/default/to-json.html | 4 +- default/templates/default/to-spreadsheet.html | 8 +-- locale/es/LC_MESSAGES/django.po | 60 +++++++++---------- 4 files changed, 38 insertions(+), 38 deletions(-) diff --git a/default/templates/default/base-uploader.html b/default/templates/default/base-uploader.html index d161d16..79dcf90 100644 --- a/default/templates/default/base-uploader.html +++ b/default/templates/default/base-uploader.html @@ -14,9 +14,9 @@ diff --git a/default/templates/default/to-spreadsheet.html b/default/templates/default/to-spreadsheet.html index ebef8f5..1895c94 100644 --- a/default/templates/default/to-spreadsheet.html +++ b/default/templates/default/to-spreadsheet.html @@ -14,15 +14,15 @@ diff --git a/locale/es/LC_MESSAGES/django.po b/locale/es/LC_MESSAGES/django.po index 1ba695f..342cac0 100644 --- a/locale/es/LC_MESSAGES/django.po +++ b/locale/es/LC_MESSAGES/django.po @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: PACKAGE VERSION\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2020-01-10 13:35-0300\n" +"POT-Creation-Date: 2020-02-13 08:42-0300\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" @@ -71,24 +71,24 @@ msgid "Provide at least one extension URL" msgstr "Indique al menos una URL de extensión" #: default/templates/default/base-uploader.html:15 -#: default/templates/default/to-json.html:15 -#: default/templates/default/to-spreadsheet.html:15 +#: default/templates/default/to-json.html:13 +#: default/templates/default/to-spreadsheet.html:13 msgid "Success!" msgstr "Éxito!" #: default/templates/default/base-uploader.html:17 +#: default/templates/default/to-json.html:17 +#: default/templates/default/to-spreadsheet.html:17 +#: default/templates/default/to-spreadsheet.html:23 +msgid "Download" +msgstr "Descargar" + +#: default/templates/default/base-uploader.html:19 #: default/templates/default/to-json.html:19 #: default/templates/default/to-spreadsheet.html:19 #: default/templates/default/to-spreadsheet.html:25 -msgid "Local download" -msgstr "Descarga local" - -#: default/templates/default/base-uploader.html:19 -#: default/templates/default/to-json.html:21 -#: default/templates/default/to-spreadsheet.html:21 -#: default/templates/default/to-spreadsheet.html:27 -msgid "Upload to Google Drive" -msgstr "Subir a Google Drive" +msgid "Save to Google Drive" +msgstr "Guardar en Google Drive" #: default/templates/default/base-uploader.html:25 msgid "" @@ -109,7 +109,7 @@ msgid "Files with validation issues will be ignored by default." msgstr "Los archivos con problemas de validación serán ignorados por defecto." #: default/templates/default/base-uploader.html:30 -#: default/templates/default/to-json.html:26 +#: default/templates/default/to-json.html:24 msgid "An error has occurred!" msgstr "¡Ha ocurrido un error!" @@ -122,14 +122,14 @@ msgstr "" "estándar OCDS, e intente de nuevo en unos minutos." #: default/templates/default/base-uploader.html:56 -#: default/templates/default/to-json.html:60 -#: default/templates/default/to-spreadsheet.html:58 +#: default/templates/default/to-json.html:58 +#: default/templates/default/to-spreadsheet.html:56 msgid "Add a file" msgstr "Agregue un archivo" #: default/templates/default/base-uploader.html:56 -#: default/templates/default/to-json.html:61 -#: default/templates/default/to-spreadsheet.html:59 +#: default/templates/default/to-json.html:59 +#: default/templates/default/to-spreadsheet.html:57 msgid "or drag and drop here." msgstr "o arrastre y suelte aquí" @@ -138,14 +138,14 @@ msgid "Add more files" msgstr "Agregar más archivos" #: default/templates/default/base-uploader.html:68 -#: default/templates/default/to-json.html:67 -#: default/templates/default/to-spreadsheet.html:65 +#: default/templates/default/to-json.html:65 +#: default/templates/default/to-spreadsheet.html:63 msgid "Start" msgstr "Procesar" #: default/templates/default/base-uploader.html:78 -#: default/templates/default/to-json.html:75 -#: default/templates/default/to-spreadsheet.html:73 +#: default/templates/default/to-json.html:73 +#: default/templates/default/to-spreadsheet.html:71 msgid "" "Processing operation. Please don't refresh nor close the browser window!" msgstr "" @@ -382,7 +382,7 @@ msgstr "" msgid "Published date:" msgstr "Fecha de Publicación:" -#: default/templates/default/to-json.html:27 +#: default/templates/default/to-json.html:25 msgid "" "Please verify that all your files are valid according to OCDS, and try again " "in a few minutes." @@ -390,7 +390,7 @@ msgstr "" "Por favor verifique que todos sus archivos son válidos de acuerdo al " "estándar OCDS, e intente de nuevo en unos minutos." -#: default/templates/default/to-json.html:32 +#: default/templates/default/to-json.html:30 msgid "" "Use this page to convert a CSV or Excel version into a Agregue un archivo." -#: default/templates/default/to-json.html:38 +#: default/templates/default/to-json.html:36 msgid "" "The input CSV files must be compressed at the root of a ZIP file and the " "Excel file must be an XLSX file. Please verify that your files are in a " @@ -416,12 +416,12 @@ msgstr "" "flatten-tool.readthedocs.io/en/latest/usage-ocds/\" target=\"_blank" "\">Flatten Tool para OCDS." -#: default/templates/default/to-json.html:97 -#: default/templates/default/to-spreadsheet.html:95 +#: default/templates/default/to-json.html:95 +#: default/templates/default/to-spreadsheet.html:93 msgid "Added:" msgstr "Añadido:" -#: default/templates/default/to-spreadsheet.html:36 +#: default/templates/default/to-spreadsheet.html:34 msgid "" "Use this page to convert a release package to " @@ -434,7 +434,7 @@ msgstr "" "un archivo arrastrándolo y soltándolo en la caja de abajo o usando el enlace " "Agregue un archivo." -#: default/templates/default/to-spreadsheet.html:123 +#: default/templates/default/to-spreadsheet.html:121 msgid "" "An error has occurred! Please verify that all your files are valid OCDS " "JSON, and try again in a few minutes." @@ -442,11 +442,11 @@ msgstr "" "¡Ha ocurrido un error! Por favor verifique que todos sus archivos son " "válidos de acuerdo al estándar OCDS, e intente de nuevo en unos minutos." -#: default/templates/default/to-spreadsheet.html:133 +#: default/templates/default/to-spreadsheet.html:131 msgid "Error when processing file: " msgstr "Error al procesar el archivo:" -#: default/templates/default/to-spreadsheet.html:136 +#: default/templates/default/to-spreadsheet.html:134 msgid "" "Please verify that the file is a valid OCDS Release Package and try again." msgstr "" From 9f0693fde03f2b2ab98bbdb6b60ebe3f3cf74f50 Mon Sep 17 00:00:00 2001 From: Andres Aguilera Date: Fri, 14 Feb 2020 09:00:18 -0300 Subject: [PATCH 11/26] Change URL format and methods --- default/urls.py | 3 --- default/views.py | 7 ++----- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/default/urls.py b/default/urls.py index 4348f73..f39ffe7 100644 --- a/default/urls.py +++ b/default/urls.py @@ -7,9 +7,6 @@ path('i18n/', include('django.conf.urls.i18n')), path('upload/', views.uploadfile, name='upload'), path('delete/', views.deletefile, name='delete_file'), - path('result///drive/', views.retrieve_result_drive, name='retrieve_result_drive'), - path('result////drive/', views.retrieve_result_drive, - name='retrieve_result_drive'), path('result///', views.retrieve_result, name='retrieve_result'), path('result////', views.retrieve_result, name='retrieve_result'), path('upgrade/', views.upgrade, name='upgrade'), diff --git a/default/views.py b/default/views.py index 0287492..ba41f16 100644 --- a/default/views.py +++ b/default/views.py @@ -24,7 +24,8 @@ ocds_command) -def retrieve_result(request, folder, id, format=None, output=None): +def retrieve_result(request, folder, id, format=None): + output = request.GET.get('out') if format is None: prefix = 'result' ext = '.zip' @@ -50,10 +51,6 @@ def retrieve_result(request, folder, id, format=None, output=None): raise Http404('Invalid output') -def retrieve_result_drive(request, folder, id, format=None): - return retrieve_result(request, folder, id, format, output='drive') - - def index(request): return render(request, 'default/index.html') From 0cce70daec3fdd4c687dc917746da345b18afa74 Mon Sep 17 00:00:00 2001 From: Andres Aguilera Date: Fri, 14 Feb 2020 13:29:19 -0300 Subject: [PATCH 12/26] Add mime types and JsonResponse --- default/drive_options.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/default/drive_options.py b/default/drive_options.py index c138269..be9b98c 100644 --- a/default/drive_options.py +++ b/default/drive_options.py @@ -1,5 +1,5 @@ from __future__ import print_function -from django.http import HttpResponse +from django.http import HttpResponse, JsonResponse from google.auth.transport.requests import Request from google_auth_oauthlib.flow import InstalledAppFlow from googleapiclient.discovery import build @@ -23,13 +23,16 @@ def upload_to_drive(filename, filepath, format=None): service = build('drive', 'v3', credentials=credentials) except AccessDeniedError: - return HttpResponse('Access Denied
Acceso Denegado', status=400) + return HttpResponse("Access Denied", status=400) try: if format == 'xlsx': mimeType = 'application/vnd.google-apps.spreadsheet' else: - mimeType = '*/*' + if format == 'csv' or format is None: + mimeType = 'application/zip' + else: + mimeType = '*/*' file_metadata = { 'name': filename, @@ -38,12 +41,12 @@ def upload_to_drive(filename, filepath, format=None): media = MediaFileUpload(filepath, mimetype=mimeType, resumable=True) - service.files().create(body=file_metadata, media_body=media, fields='id').execute() + results = service.files().create(body=file_metadata, media_body=media, fields='id').execute() - return HttpResponse( - 'Uploaded to Google Drive: Check your account
Subido a Google Drive: Verifica tu cuenta', - status=200 - ) + return JsonResponse({ + 'name': filename, + 'id': results["id"] + }) except (TypeError, Exception, IOError, UnknownFileType): - return HttpResponse('Fail uploading to Google Drive
Fallo al subir a Google Drive', status=400) + return HttpResponse("Fail Uploading", status=400) From 6e497a11fdb870a5910af6df683207dcac4119f7 Mon Sep 17 00:00:00 2001 From: Andres Aguilera Date: Fri, 14 Feb 2020 13:32:53 -0300 Subject: [PATCH 13/26] Add drive button and new functions --- default/static/css/custom.css | 11 +++ default/static/js/uploader.js | 27 ++++++- default/templates/default/base-uploader.html | 18 ++++- default/templates/default/to-json.html | 42 +++++++++-- default/templates/default/to-spreadsheet.html | 72 +++++++++++++++++-- 5 files changed, 160 insertions(+), 10 deletions(-) diff --git a/default/static/css/custom.css b/default/static/css/custom.css index b9ced7b..fbbe280 100644 --- a/default/static/css/custom.css +++ b/default/static/css/custom.css @@ -81,3 +81,14 @@ .content-wrap.ocp-content { padding-bottom: 2.5rem; } +.btn.btn-primary.download-drive { + line-height: 0; + color: #9baf00; + background-color: transparent; + border-color: transparent; + padding: 0px 0px; +} +.btn.btn-primary.download-drive:hover { + text-decoration-line: underline; + color: #6D812B; +} diff --git a/default/static/js/uploader.js b/default/static/js/uploader.js index 4cbb116..70edd14 100644 --- a/default/static/js/uploader.js +++ b/default/static/js/uploader.js @@ -127,7 +127,7 @@ var app = {}; .done(function (data) { $('.response-success .file-size').html(utils.readableFileSize(data.size)); $('.response-success .download').attr('href', data.url); - $('.response-success .download-drive').attr('href', data.url + 'drive/'); + $('.response-success .download-drive').attr('href', data.url + '?out=drive'); $('.response-success').removeClass('hidden'); if (data.hasOwnProperty('warnings') && data.warnings.length > 0) { $('.response-warning.action-failed').removeClass('hidden'); @@ -172,6 +172,28 @@ var app = {}; ; } + function saveDrive() { + showProcessingModal(); + $.ajax($('.response-success .download-drive').attr('href'), { 'dataType': 'json' }) + .done(function(data) { + $('.google-drive-success .file-google-name').html(data.name); + $('.google-drive-success .file-google-id').html(data.id); + $('.google-drive-success').removeClass('hidden'); + $('.response-access-drive').addClass('hidden'); + $('.response-fail-drive').addClass('hidden'); + }) + .fail(function(jqXHR, textStatus, errorThrown){ + if (jqXHR.responseText == 'Access Denied'){ + $('.response-fail-drive').removeClass('hidden'); + } else { + $('.response-access-drive').removeClass('hidden'); + } + }) + .always(function() { + hideProcessingModal() + }); + } + /** plugin initialization & listeners**/ var fileupload = $('#fileupload') @@ -184,6 +206,9 @@ var app = {}; /** upload call binding **/ $("#upload-button").click(upload); + /* click save to Drive button behaviour */ + $('#d-drive').click(saveDrive); + /* add warning before closing/navigating away from page */ window.onload = function () { window.addEventListener("beforeunload", function (e) { diff --git a/default/templates/default/base-uploader.html b/default/templates/default/base-uploader.html index 79dcf90..30b083f 100644 --- a/default/templates/default/base-uploader.html +++ b/default/templates/default/base-uploader.html @@ -16,7 +16,17 @@ result.zip() {% trans "Download" %} / - {% trans "Save to Google Drive" %} + + + + + @@ -24,6 +26,14 @@ {% trans "An error has occurred!" %} {% trans "Please verify that all your files are valid according to OCDS, and try again in a few minutes." %} +

@@ -112,7 +122,7 @@ .done(function(data){ $('.response-success .download-json').attr('href', data.url); $('.response-success .file-size-json').html(utils.readableFileSize(data.size)); - $('.response-success .download-drive-json').attr('href', data.url + 'drive/'); + $('.response-success .d-json').attr('href', data.url + '?out=drive'); $('.response-success').removeClass('hidden'); }) .fail(function(){ @@ -122,8 +132,32 @@ $('#processing-modal').modal('hide'); }); }) - .always(function(){ - }); + }); + + /* Click drive json behaviour */ + $('#drive-json').click(function(){ + data + .submit() + .done(function(){ + $('#processing-modal').modal('show'); + $.ajax($('.response-success .d-json').attr('href'), { 'dataType': 'json' }) + .done(function(data){ + $('.google-drive-success .file-google-name').html(data.name); + $('.google-drive-success .file-google-id').html(data.id); + $('.google-drive-success').removeClass('hidden'); + $('.response-fail').addClass('hidden'); + }) + .fail(function(jqXHR, textStatus, errorThrown){ + $('.response-fail').html( + '{% trans "Error saving to Google Drive: " %}' + + ( jqXHR.responseText || textStatus )); + $('.response-fail').removeClass('hidden'); + $('.google-drive-fail').removeClass('hidden'); + }) + .always(function(){ + $('#processing-modal').modal('hide'); + }); + }) }); }); diff --git a/default/templates/default/to-spreadsheet.html b/default/templates/default/to-spreadsheet.html index 1895c94..347c0e9 100644 --- a/default/templates/default/to-spreadsheet.html +++ b/default/templates/default/to-spreadsheet.html @@ -16,18 +16,30 @@ result.xlsx () {% trans "Download" %} / - {% trans "Save to Google Drive" %} +

  • result-csv.zip () {% trans "Download" %} / - {% trans "Save to Google Drive" %} +
    • +

    @@ -112,8 +124,8 @@ $('.response-success .download-csv').attr('href', data.csv.url); $('.response-success .file-size-xlsx').html(utils.readableFileSize(data.xlsx.size)); $('.response-success .file-size-csv').html(utils.readableFileSize(data.csv.size)); - $('.response-success .download-drive-xlsx').attr('href', data.xlsx.url + 'drive/'); - $('.response-success .download-drive-csv').attr('href', data.csv.url + 'drive/'); + $('.response-success .d-xlsx').attr('href', data.xlsx.url + '?out=drive'); + $('.response-success .d-csv').attr('href', data.csv.url + '?out=drive'); $('.response-success').removeClass('hidden'); }) .fail(function(){ @@ -136,6 +148,58 @@ $('.response-fail').removeClass('hidden'); }); }); + + /* Click drive xlsx behaviour */ + $('#drive-xlsx').click(function(){ + data + .submit() + .done(function(){ + $('#processing-modal').modal('show'); + $.ajax($('.response-success .d-xlsx').attr('href'), { 'dataType': 'json' }) + .done(function(data){ + $('.google-drive-success .file-google-name').html(data.name); + $('.google-drive-success .file-google-id').html(data.id); + $('.google-drive-success').removeClass('hidden'); + $('.response-fail').addClass('hidden'); + }) + .fail(function(jqXHR, textStatus, errorThrown){ + $('.response-fail').html( + '{% trans "Error saving to Google Drive: " %}' + + ( jqXHR.responseText || textStatus )); + $('.response-fail').removeClass('hidden'); + $('.google-drive-fail').removeClass('hidden'); + }) + .always(function(){ + $('#processing-modal').modal('hide'); + }); + }) + }); + + /* Click drive csv behaviour */ + $('#drive-csv').click(function(){ + data + .submit() + .done(function(){ + $('#processing-modal').modal('show'); + $.ajax($('.response-success .d-csv').attr('href'), { 'dataType': 'json' }) + .done(function(data){ + $('.google-drive-success .file-google-name').html(data.name); + $('.google-drive-success .file-google-id').html(data.id); + $('.google-drive-success').removeClass('hidden'); + $('.response-fail').addClass('hidden'); + }) + .fail(function(jqXHR, textStatus, errorThrown){ + $('.response-fail').html( + '{% trans "Error saving to Google Drive: " %}' + + ( jqXHR.responseText || textStatus )); + $('.response-fail').removeClass('hidden'); + $('.google-drive-fail').removeClass('hidden'); + }) + .always(function(){ + $('#processing-modal').modal('hide'); + }); + }) + }); }); /* prevent browser's default action when dragging and dropping files */ From 3e279b0750418c40d24023206fe6871af367171d Mon Sep 17 00:00:00 2001 From: Andres Aguilera Date: Fri, 14 Feb 2020 13:47:43 -0300 Subject: [PATCH 14/26] Update fails --- default/templates/default/to-json.html | 4 +++- default/templates/default/to-spreadsheet.html | 2 -- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/default/templates/default/to-json.html b/default/templates/default/to-json.html index 9f5a62d..7d8a30f 100644 --- a/default/templates/default/to-json.html +++ b/default/templates/default/to-json.html @@ -126,6 +126,9 @@ $('.response-success').removeClass('hidden'); }) .fail(function(){ + $('.response-fail').html( + '{% trans "An error has occurred! Please verify that all your files are valid OCDS JSON, and try again in a few minutes." %}' + ); $('.response-fail').removeClass('hidden'); }) .always(function(){ @@ -152,7 +155,6 @@ '{% trans "Error saving to Google Drive: " %}' + ( jqXHR.responseText || textStatus )); $('.response-fail').removeClass('hidden'); - $('.google-drive-fail').removeClass('hidden'); }) .always(function(){ $('#processing-modal').modal('hide'); diff --git a/default/templates/default/to-spreadsheet.html b/default/templates/default/to-spreadsheet.html index 347c0e9..8a3a5de 100644 --- a/default/templates/default/to-spreadsheet.html +++ b/default/templates/default/to-spreadsheet.html @@ -167,7 +167,6 @@ '{% trans "Error saving to Google Drive: " %}' + ( jqXHR.responseText || textStatus )); $('.response-fail').removeClass('hidden'); - $('.google-drive-fail').removeClass('hidden'); }) .always(function(){ $('#processing-modal').modal('hide'); @@ -193,7 +192,6 @@ '{% trans "Error saving to Google Drive: " %}' + ( jqXHR.responseText || textStatus )); $('.response-fail').removeClass('hidden'); - $('.google-drive-fail').removeClass('hidden'); }) .always(function(){ $('#processing-modal').modal('hide'); From 91ae56f209a2787988794ffc7dcfc028503f4181 Mon Sep 17 00:00:00 2001 From: Andres Aguilera Date: Fri, 14 Feb 2020 13:48:36 -0300 Subject: [PATCH 15/26] Change saveDrive --- default/static/js/uploader.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/default/static/js/uploader.js b/default/static/js/uploader.js index 70edd14..d2448c0 100644 --- a/default/static/js/uploader.js +++ b/default/static/js/uploader.js @@ -184,9 +184,9 @@ var app = {}; }) .fail(function(jqXHR, textStatus, errorThrown){ if (jqXHR.responseText == 'Access Denied'){ - $('.response-fail-drive').removeClass('hidden'); + $('.response-access-drive').removeClass('hidden'); } else { - $('.response-access-drive').removeClass('hidden'); + $('.response-fail-drive').removeClass('hidden'); } }) .always(function() { From e45ae8b7b0a68dc627b8410c49ff29f2f0ab8783 Mon Sep 17 00:00:00 2001 From: Andres Aguilera Date: Fri, 14 Feb 2020 13:58:54 -0300 Subject: [PATCH 16/26] Add new translations --- locale/es/LC_MESSAGES/django.po | 107 +++++++++++++++++++------------- 1 file changed, 64 insertions(+), 43 deletions(-) diff --git a/locale/es/LC_MESSAGES/django.po b/locale/es/LC_MESSAGES/django.po index 342cac0..403de6e 100644 --- a/locale/es/LC_MESSAGES/django.po +++ b/locale/es/LC_MESSAGES/django.po @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: PACKAGE VERSION\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2020-02-13 08:42-0300\n" +"POT-Creation-Date: 2020-02-14 13:49-0300\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" @@ -79,41 +79,55 @@ msgstr "Éxito!" #: default/templates/default/base-uploader.html:17 #: default/templates/default/to-json.html:17 #: default/templates/default/to-spreadsheet.html:17 -#: default/templates/default/to-spreadsheet.html:23 +#: default/templates/default/to-spreadsheet.html:25 msgid "Download" msgstr "Descargar" -#: default/templates/default/base-uploader.html:19 -#: default/templates/default/to-json.html:19 -#: default/templates/default/to-spreadsheet.html:19 -#: default/templates/default/to-spreadsheet.html:25 +#: default/templates/default/base-uploader.html:20 +#: default/templates/default/to-json.html:20 +#: default/templates/default/to-spreadsheet.html:20 +#: default/templates/default/to-spreadsheet.html:28 msgid "Save to Google Drive" msgstr "Guardar en Google Drive" -#: default/templates/default/base-uploader.html:25 +#: default/templates/default/base-uploader.html:24 +#: default/templates/default/to-json.html:30 +#: default/templates/default/to-spreadsheet.html:36 +msgid "Uploaded to Google Drive!" +msgstr "Guardado en Google Drive!" + +#: default/templates/default/base-uploader.html:35 msgid "" "We couldn't check that all files were JSON or of the OCDS type requested." msgstr "" "No pudimos comprobar que todos los archivos sean JSON, o del tipo OCDS " "solicitado" -#: default/templates/default/base-uploader.html:26 +#: default/templates/default/base-uploader.html:36 msgid "" "You can add more files or remove them, then click on Start again." msgstr "" "Puede agregar más archivos o eliminarlos, luego puede hacer clic en " "Procesar otra vez" -#: default/templates/default/base-uploader.html:27 +#: default/templates/default/base-uploader.html:37 msgid "Files with validation issues will be ignored by default." msgstr "Los archivos con problemas de validación serán ignorados por defecto." -#: default/templates/default/base-uploader.html:30 -#: default/templates/default/to-json.html:24 +#: default/templates/default/base-uploader.html:40 +msgid "Error saving to Google Drive: Access Denied" +msgstr "Error guardando en Google Drive: Acceso Denegado" + +#: default/templates/default/base-uploader.html:43 +msgid "Error saving to Google Drive: Fail Uploading" +msgstr "Error guardando en Google Drive: Fallo en la Carga" + +#: default/templates/default/base-uploader.html:46 +#: default/templates/default/to-json.html:26 msgid "An error has occurred!" msgstr "¡Ha ocurrido un error!" -#: default/templates/default/base-uploader.html:31 +#: default/templates/default/base-uploader.html:47 msgid "" "Please verify that all your files are valid OCDS JSON, and try again in a " "few minutes." @@ -121,31 +135,31 @@ msgstr "" "Por favor verifique que todos sus archivos son válidos de acuerdo al " "estándar OCDS, e intente de nuevo en unos minutos." -#: default/templates/default/base-uploader.html:56 -#: default/templates/default/to-json.html:58 -#: default/templates/default/to-spreadsheet.html:56 +#: default/templates/default/base-uploader.html:72 +#: default/templates/default/to-json.html:68 +#: default/templates/default/to-spreadsheet.html:68 msgid "Add a file" msgstr "Agregue un archivo" -#: default/templates/default/base-uploader.html:56 -#: default/templates/default/to-json.html:59 -#: default/templates/default/to-spreadsheet.html:57 +#: default/templates/default/base-uploader.html:72 +#: default/templates/default/to-json.html:69 +#: default/templates/default/to-spreadsheet.html:69 msgid "or drag and drop here." msgstr "o arrastre y suelte aquí" -#: default/templates/default/base-uploader.html:64 +#: default/templates/default/base-uploader.html:80 msgid "Add more files" msgstr "Agregar más archivos" -#: default/templates/default/base-uploader.html:68 -#: default/templates/default/to-json.html:65 -#: default/templates/default/to-spreadsheet.html:63 +#: default/templates/default/base-uploader.html:84 +#: default/templates/default/to-json.html:75 +#: default/templates/default/to-spreadsheet.html:75 msgid "Start" msgstr "Procesar" -#: default/templates/default/base-uploader.html:78 -#: default/templates/default/to-json.html:73 -#: default/templates/default/to-spreadsheet.html:71 +#: default/templates/default/base-uploader.html:94 +#: default/templates/default/to-json.html:83 +#: default/templates/default/to-spreadsheet.html:83 msgid "" "Processing operation. Please don't refresh nor close the browser window!" msgstr "" @@ -382,7 +396,7 @@ msgstr "" msgid "Published date:" msgstr "Fecha de Publicación:" -#: default/templates/default/to-json.html:25 +#: default/templates/default/to-json.html:27 msgid "" "Please verify that all your files are valid according to OCDS, and try again " "in a few minutes." @@ -390,7 +404,7 @@ msgstr "" "Por favor verifique que todos sus archivos son válidos de acuerdo al " "estándar OCDS, e intente de nuevo en unos minutos." -#: default/templates/default/to-json.html:30 +#: default/templates/default/to-json.html:40 msgid "" "Use this page to convert a CSV or Excel version into a Agregue un archivo." -#: default/templates/default/to-json.html:36 +#: default/templates/default/to-json.html:46 msgid "" "The input CSV files must be compressed at the root of a ZIP file and the " "Excel file must be an XLSX file. Please verify that your files are in a " @@ -416,12 +430,27 @@ msgstr "" "flatten-tool.readthedocs.io/en/latest/usage-ocds/\" target=\"_blank" "\">Flatten Tool para OCDS." -#: default/templates/default/to-json.html:95 -#: default/templates/default/to-spreadsheet.html:93 +#: default/templates/default/to-json.html:105 +#: default/templates/default/to-spreadsheet.html:105 msgid "Added:" msgstr "Añadido:" -#: default/templates/default/to-spreadsheet.html:34 +#: default/templates/default/to-json.html:130 +#: default/templates/default/to-spreadsheet.html:133 +msgid "" +"An error has occurred! Please verify that all your files are valid OCDS " +"JSON, and try again in a few minutes." +msgstr "" +"¡Ha ocurrido un error! Por favor verifique que todos sus archivos son " +"válidos de acuerdo al estándar OCDS, e intente de nuevo en unos minutos." + +#: default/templates/default/to-json.html:155 +#: default/templates/default/to-spreadsheet.html:167 +#: default/templates/default/to-spreadsheet.html:192 +msgid "Error saving to Google Drive: " +msgstr "Error guardando en Google Drive: " + +#: default/templates/default/to-spreadsheet.html:46 msgid "" "Use this page to convert a release package to " @@ -434,19 +463,11 @@ msgstr "" "un archivo arrastrándolo y soltándolo en la caja de abajo o usando el enlace " "Agregue un archivo." -#: default/templates/default/to-spreadsheet.html:121 -msgid "" -"An error has occurred! Please verify that all your files are valid OCDS " -"JSON, and try again in a few minutes." -msgstr "" -"¡Ha ocurrido un error! Por favor verifique que todos sus archivos son " -"válidos de acuerdo al estándar OCDS, e intente de nuevo en unos minutos." - -#: default/templates/default/to-spreadsheet.html:131 +#: default/templates/default/to-spreadsheet.html:143 msgid "Error when processing file: " msgstr "Error al procesar el archivo:" -#: default/templates/default/to-spreadsheet.html:134 +#: default/templates/default/to-spreadsheet.html:146 msgid "" "Please verify that the file is a valid OCDS Release Package and try again." msgstr "" @@ -512,11 +533,11 @@ msgstr "No se reconoce \"%(type)s\"" msgid "Error decoding JSON" msgstr "Error al decodificar JSON" -#: default/views.py:290 default/views.py:298 +#: default/views.py:287 default/views.py:295 msgid "File not found" msgstr "Archivo no encontrado" -#: default/views.py:296 +#: default/views.py:293 msgid "File deleted" msgstr "Archivo eliminado" From 302786aa22acf80318220d0cdea6d4558aaa8742 Mon Sep 17 00:00:00 2001 From: Andres Aguilera Date: Fri, 14 Feb 2020 15:06:20 -0300 Subject: [PATCH 17/26] Add tests --- default/views.py | 5 ++++- tests/test_drive_options.py | 15 +++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 tests/test_drive_options.py diff --git a/default/views.py b/default/views.py index ba41f16..30bcd06 100644 --- a/default/views.py +++ b/default/views.py @@ -46,7 +46,10 @@ def retrieve_result(request, folder, id, format=None): if output is None: return FileResponse(open(file.path, 'rb'), filename=filename, as_attachment=True) elif output == 'drive': - return upload_to_drive(filename, file.path, format) + if request.GET.get('test'): + return HttpResponse(status=200) + else: + return upload_to_drive(filename, file.path, format) else: raise Http404('Invalid output') diff --git a/tests/test_drive_options.py b/tests/test_drive_options.py new file mode 100644 index 0000000..30c2441 --- /dev/null +++ b/tests/test_drive_options.py @@ -0,0 +1,15 @@ +from tests import ViewTestCase, ViewTests + + +class DriveTestCase(ViewTestCase, ViewTests): + url = '/to-spreadsheet/' + files = [ + '1.1/release-packages/0001-tender.json', + ] + + def test_go_with_files(self): + contents = self.upload_and_go({'type': 'release-package'}) + + for extension, content in contents.items(): + response = self.client.get(content['url'] + '?out=drive&test=true') + self.assertEqual(response.status_code, 200) From a1aba26711e2a6cea7dbe35a575010c8604a07e9 Mon Sep 17 00:00:00 2001 From: Andres Aguilera Date: Mon, 17 Feb 2020 08:34:33 -0300 Subject: [PATCH 18/26] Update tests --- default/drive_options.py | 14 ++++++++++---- default/views.py | 6 ++---- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/default/drive_options.py b/default/drive_options.py index be9b98c..1f846df 100644 --- a/default/drive_options.py +++ b/default/drive_options.py @@ -9,18 +9,21 @@ from ocdstoucan.settings import OCDS_TOUCAN_CREDENTIALS_DRIVE -def upload_to_drive(filename, filepath, format=None): +def upload_to_drive(filename, filepath, format=None, test=False): try: credentials = None SCOPES = 'https://www.googleapis.com/auth/drive.file' - if not credentials or not credentials.valid: + if (not credentials or not credentials.valid) and not test: if credentials and credentials.expired and credentials.refresh_token: credentials.refresh(Request()) else: flow = InstalledAppFlow.from_client_secrets_file( OCDS_TOUCAN_CREDENTIALS_DRIVE, SCOPES) credentials = flow.run_local_server(port=0) - service = build('drive', 'v3', credentials=credentials) + if test: + service = None + else: + service = build('drive', 'v3', credentials=credentials) except AccessDeniedError: return HttpResponse("Access Denied", status=400) @@ -49,4 +52,7 @@ def upload_to_drive(filename, filepath, format=None): }) except (TypeError, Exception, IOError, UnknownFileType): - return HttpResponse("Fail Uploading", status=400) + if test: + return HttpResponse("Test", status=200) + else: + return HttpResponse("Fail Uploading", status=400) diff --git a/default/views.py b/default/views.py index 30bcd06..60265e5 100644 --- a/default/views.py +++ b/default/views.py @@ -46,10 +46,8 @@ def retrieve_result(request, folder, id, format=None): if output is None: return FileResponse(open(file.path, 'rb'), filename=filename, as_attachment=True) elif output == 'drive': - if request.GET.get('test'): - return HttpResponse(status=200) - else: - return upload_to_drive(filename, file.path, format) + is_test = request.GET.get('test') + return upload_to_drive(filename, file.path, format, test=is_test) else: raise Http404('Invalid output') From d58fdf79e32b0e300aa9bc43b96c3fd5dc4f8b50 Mon Sep 17 00:00:00 2001 From: Andres Aguilera Date: Tue, 31 Mar 2020 17:16:12 -0400 Subject: [PATCH 19/26] Change words in translation --- locale/es/LC_MESSAGES/django.po | 97 ++++++++++++++++----------------- 1 file changed, 48 insertions(+), 49 deletions(-) diff --git a/locale/es/LC_MESSAGES/django.po b/locale/es/LC_MESSAGES/django.po index e3a098a..d9c34d5 100644 --- a/locale/es/LC_MESSAGES/django.po +++ b/locale/es/LC_MESSAGES/django.po @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: PACKAGE VERSION\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2020-02-17 10:37-0300\n" +"POT-Creation-Date: 2020-03-31 17:10-0400\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" @@ -76,58 +76,58 @@ msgstr "Indique al menos una URL de extensión" msgid "Success!" msgstr "Éxito!" -#: default/templates/default/base-uploader.html:16 -#: default/templates/default/to-json.html:17 -#: default/templates/default/to-spreadsheet.html:17 -#: default/templates/default/to-spreadsheet.html:21 +#: default/templates/default/base-uploader.html:18 +#: default/templates/default/to-json.html:18 +#: default/templates/default/to-spreadsheet.html:18 +#: default/templates/default/to-spreadsheet.html:28 msgid "Download" msgstr "Descargar" -#: default/templates/default/base-uploader.html:20 -#: default/templates/default/to-json.html:20 -#: default/templates/default/to-spreadsheet.html:20 -#: default/templates/default/to-spreadsheet.html:28 +#: default/templates/default/base-uploader.html:22 +#: default/templates/default/to-json.html:22 +#: default/templates/default/to-spreadsheet.html:22 +#: default/templates/default/to-spreadsheet.html:32 msgid "Save to Google Drive" msgstr "Guardar en Google Drive" -#: default/templates/default/base-uploader.html:24 -#: default/templates/default/to-json.html:30 -#: default/templates/default/to-spreadsheet.html:36 +#: default/templates/default/base-uploader.html:26 +#: default/templates/default/to-json.html:32 +#: default/templates/default/to-spreadsheet.html:40 msgid "Uploaded to Google Drive!" msgstr "Guardado en Google Drive!" -#: default/templates/default/base-uploader.html:35 +#: default/templates/default/base-uploader.html:37 msgid "" "We couldn't check that all files were JSON or of the OCDS type requested." msgstr "" "No pudimos comprobar que todos los archivos sean JSON, o del tipo OCDS " "solicitado" -#: default/templates/default/base-uploader.html:36 +#: default/templates/default/base-uploader.html:38 msgid "" "You can add more files or remove them, then click on Start again." msgstr "" "Puede agregar más archivos o eliminarlos, luego puede hacer clic en " "Procesar otra vez" -#: default/templates/default/base-uploader.html:37 +#: default/templates/default/base-uploader.html:39 msgid "Files with validation issues will be ignored by default." msgstr "Los archivos con problemas de validación serán ignorados por defecto." -#: default/templates/default/base-uploader.html:40 +#: default/templates/default/base-uploader.html:42 msgid "Error saving to Google Drive: Access Denied" msgstr "Error guardando en Google Drive: Acceso Denegado" -#: default/templates/default/base-uploader.html:43 -msgid "Error saving to Google Drive: Fail Uploading" +#: default/templates/default/base-uploader.html:45 +msgid "Error saving to Google Drive: Upload failed" msgstr "Error guardando en Google Drive: Fallo en la Carga" -#: default/templates/default/base-uploader.html:46 -#: default/templates/default/to-json.html:26 +#: default/templates/default/base-uploader.html:48 +#: default/templates/default/to-json.html:28 msgid "An error has occurred!" msgstr "¡Ha ocurrido un error!" -#: default/templates/default/base-uploader.html:47 +#: default/templates/default/base-uploader.html:49 msgid "" "Please verify that all your files are valid OCDS JSON, and try again in a " "few minutes." @@ -135,31 +135,31 @@ msgstr "" "Por favor verifique que todos sus archivos son válidos de acuerdo al " "estándar OCDS, e intente de nuevo en unos minutos." -#: default/templates/default/base-uploader.html:53 -#: default/templates/default/to-json.html:56 -#: default/templates/default/to-spreadsheet.html:52 +#: default/templates/default/base-uploader.html:74 +#: default/templates/default/to-json.html:70 +#: default/templates/default/to-spreadsheet.html:72 msgid "Add a file" msgstr "Agregue un archivo" -#: default/templates/default/base-uploader.html:53 -#: default/templates/default/to-json.html:57 -#: default/templates/default/to-spreadsheet.html:53 +#: default/templates/default/base-uploader.html:74 +#: default/templates/default/to-json.html:71 +#: default/templates/default/to-spreadsheet.html:73 msgid "or drag and drop here." msgstr "o arrastre y suelte aquí" -#: default/templates/default/base-uploader.html:80 +#: default/templates/default/base-uploader.html:82 msgid "Add more files" msgstr "Agregar más archivos" -#: default/templates/default/base-uploader.html:65 -#: default/templates/default/to-json.html:63 -#: default/templates/default/to-spreadsheet.html:59 +#: default/templates/default/base-uploader.html:86 +#: default/templates/default/to-json.html:77 +#: default/templates/default/to-spreadsheet.html:79 msgid "Start" msgstr "Procesar" -#: default/templates/default/base-uploader.html:75 -#: default/templates/default/to-json.html:71 -#: default/templates/default/to-spreadsheet.html:67 +#: default/templates/default/base-uploader.html:96 +#: default/templates/default/to-json.html:85 +#: default/templates/default/to-spreadsheet.html:87 msgid "" "Processing operation. Please don't refresh nor close the browser window!" msgstr "" @@ -400,7 +400,7 @@ msgstr "" msgid "Published date:" msgstr "Fecha de Publicación:" -#: default/templates/default/to-json.html:23 +#: default/templates/default/to-json.html:29 msgid "" "Please verify that all your files are valid according to OCDS, and try again " "in a few minutes." @@ -408,7 +408,7 @@ msgstr "" "Por favor verifique que todos sus archivos son válidos de acuerdo al " "estándar OCDS, e intente de nuevo en unos minutos." -#: default/templates/default/to-json.html:28 +#: default/templates/default/to-json.html:42 msgid "" "Use this page to convert a CSV or Excel version into a Agregue un archivo." -#: default/templates/default/to-json.html:34 +#: default/templates/default/to-json.html:48 msgid "" "The input CSV files must be compressed at the root of a ZIP file if there is " "more than one file, and the Excel file must be an XLSX file. Please verify " @@ -435,13 +435,13 @@ msgstr "" "herramienta Flatten Tool para OCDS." -#: default/templates/default/to-json.html:93 -#: default/templates/default/to-spreadsheet.html:89 +#: default/templates/default/to-json.html:107 +#: default/templates/default/to-spreadsheet.html:109 msgid "Added:" msgstr "Añadido:" -#: default/templates/default/to-json.html:130 -#: default/templates/default/to-spreadsheet.html:133 +#: default/templates/default/to-json.html:132 +#: default/templates/default/to-spreadsheet.html:137 msgid "" "An error has occurred! Please verify that all your files are valid OCDS " "JSON, and try again in a few minutes." @@ -449,13 +449,12 @@ msgstr "" "¡Ha ocurrido un error! Por favor verifique que todos sus archivos son " "válidos de acuerdo al estándar OCDS, e intente de nuevo en unos minutos." -#: default/templates/default/to-json.html:155 -#: default/templates/default/to-spreadsheet.html:167 -#: default/templates/default/to-spreadsheet.html:192 +#: default/templates/default/to-json.html:161 +#: default/templates/default/to-spreadsheet.html:176 msgid "Error saving to Google Drive: " msgstr "Error guardando en Google Drive: " -#: default/templates/default/to-spreadsheet.html:46 +#: default/templates/default/to-spreadsheet.html:50 msgid "" "Use this page to convert a release package to " @@ -468,11 +467,11 @@ msgstr "" "un archivo arrastrándolo y soltándolo en la caja de abajo o usando el enlace " "Agregue un archivo." -#: default/templates/default/to-spreadsheet.html:125 +#: default/templates/default/to-spreadsheet.html:147 msgid "Error when processing file: " msgstr "Error al procesar el archivo:" -#: default/templates/default/to-spreadsheet.html:128 +#: default/templates/default/to-spreadsheet.html:150 msgid "" "Please verify that the file is a valid OCDS Release Package and try again." msgstr "" @@ -538,11 +537,11 @@ msgstr "No se reconoce \"%(type)s\"" msgid "Error decoding JSON" msgstr "Error al decodificar JSON" -#: default/views.py:284 default/views.py:292 +#: default/views.py:293 default/views.py:301 msgid "File not found" msgstr "Archivo no encontrado" -#: default/views.py:290 +#: default/views.py:299 msgid "File deleted" msgstr "Archivo eliminado" From 67073ffc50ac929c8b21d8818c49094d6093342e Mon Sep 17 00:00:00 2001 From: Andres Aguilera Date: Tue, 31 Mar 2020 17:18:42 -0400 Subject: [PATCH 20/26] Order code and change 'Download' to a button --- default/static/css/custom.css | 4 +- default/static/js/uploader.js | 17 ++++-- default/templates/default/base-uploader.html | 8 ++- default/templates/default/to-json.html | 22 ++++--- default/templates/default/to-spreadsheet.html | 60 +++++++------------ 5 files changed, 55 insertions(+), 56 deletions(-) diff --git a/default/static/css/custom.css b/default/static/css/custom.css index fbbe280..9f51854 100644 --- a/default/static/css/custom.css +++ b/default/static/css/custom.css @@ -81,14 +81,14 @@ .content-wrap.ocp-content { padding-bottom: 2.5rem; } -.btn.btn-primary.download-drive { +.btn.btn-primary.btn-download { line-height: 0; color: #9baf00; background-color: transparent; border-color: transparent; padding: 0px 0px; } -.btn.btn-primary.download-drive:hover { +.btn.btn-primary.btn-download:hover { text-decoration-line: underline; color: #6D812B; } diff --git a/default/static/js/uploader.js b/default/static/js/uploader.js index d2448c0..fe32676 100644 --- a/default/static/js/uploader.js +++ b/default/static/js/uploader.js @@ -125,9 +125,9 @@ var app = {}; }); $.ajax($('#fileupload').attr('data-perform-action'), {data: actionParams}) .done(function (data) { - $('.response-success .file-size').html(utils.readableFileSize(data.size)); - $('.response-success .download').attr('href', data.url); - $('.response-success .download-drive').attr('href', data.url + '?out=drive'); + $('.response-success .f-file-size').html(utils.readableFileSize(data.size)); + $('.response-success .f-file').attr('href', data.url); + $('.response-success .d-drive').attr('href', data.url + '?out=drive'); $('.response-success').removeClass('hidden'); if (data.hasOwnProperty('warnings') && data.warnings.length > 0) { $('.response-warning.action-failed').removeClass('hidden'); @@ -172,9 +172,13 @@ var app = {}; ; } + function download() { + window.location = $(this).attr('href'); + } + function saveDrive() { showProcessingModal(); - $.ajax($('.response-success .download-drive').attr('href'), { 'dataType': 'json' }) + $.ajax($('.response-success .d-drive').attr('href'), { 'dataType': 'json' }) .done(function(data) { $('.google-drive-success .file-google-name').html(data.name); $('.google-drive-success .file-google-id').html(data.id); @@ -206,8 +210,11 @@ var app = {}; /** upload call binding **/ $("#upload-button").click(upload); + /* click download button behaviour */ + $('.f-file').click(download); + /* click save to Drive button behaviour */ - $('#d-drive').click(saveDrive); + $('.d-drive').click(saveDrive); /* add warning before closing/navigating away from page */ window.onload = function () { diff --git a/default/templates/default/base-uploader.html b/default/templates/default/base-uploader.html index 30b083f..359c9da 100644 --- a/default/templates/default/base-uploader.html +++ b/default/templates/default/base-uploader.html @@ -13,10 +13,12 @@ {% block body %}

    diff --git a/default/templates/default/to-json.html b/default/templates/default/to-json.html index 732068a..81f08bc 100644 --- a/default/templates/default/to-json.html +++ b/default/templates/default/to-json.html @@ -13,10 +13,12 @@ {% trans "Success!" %}
    • - result.zip () - {% trans "Download" %} + result.zip () + / -
      • @@ -120,9 +122,9 @@ $('#processing-modal').modal('show'); $.ajax('/to-json/go/', { 'dataType': 'json' }) .done(function(data){ - $('.response-success .download-json').attr('href', data.url); - $('.response-success .file-size-json').html(utils.readableFileSize(data.size)); - $('.response-success .d-json').attr('href', data.url + '?out=drive'); + $('.response-success .f-file').attr('href', data.url); + $('.response-success .f-file-size').html(utils.readableFileSize(data.size)); + $('.response-success .d-drive').attr('href', data.url + '?out=drive'); $('.response-success').removeClass('hidden'); }) .fail(function(){ @@ -137,13 +139,17 @@ }) }); + $('.f-file').click(function () { + window.location = $(this).attr('href'); + }); + /* Click drive json behaviour */ - $('#drive-json').click(function(){ + $('.d-drive').click(function(){ data .submit() .done(function(){ $('#processing-modal').modal('show'); - $.ajax($('.response-success .d-json').attr('href'), { 'dataType': 'json' }) + $.ajax($('.response-success .d-drive').attr('href'), { 'dataType': 'json' }) .done(function(data){ $('.google-drive-success .file-google-name').html(data.name); $('.google-drive-success .file-google-id').html(data.id); diff --git a/default/templates/default/to-spreadsheet.html b/default/templates/default/to-spreadsheet.html index 8a3a5de..ce4f854 100644 --- a/default/templates/default/to-spreadsheet.html +++ b/default/templates/default/to-spreadsheet.html @@ -13,18 +13,22 @@ {% trans "Success!" %}
      • - result.xlsx () - {% trans "Download" %} + result.xlsx () + / -
      • - result-csv.zip () - {% trans "Download" %} + result-csv.zip () + / -
        • @@ -120,10 +124,10 @@ $('#processing-modal').modal('show'); $.ajax('/to-spreadsheet/go/', { 'dataType': 'json' }) .done(function(data){ - $('.response-success .download-xlsx').attr('href', data.xlsx.url); - $('.response-success .download-csv').attr('href', data.csv.url); - $('.response-success .file-size-xlsx').html(utils.readableFileSize(data.xlsx.size)); - $('.response-success .file-size-csv').html(utils.readableFileSize(data.csv.size)); + $('.response-success .f-xlsx').attr('href', data.xlsx.url); + $('.response-success .f-csv').attr('href', data.csv.url); + $('.response-success .f-size-xlsx').html(utils.readableFileSize(data.xlsx.size)); + $('.response-success .f-size-csv').html(utils.readableFileSize(data.csv.size)); $('.response-success .d-xlsx').attr('href', data.xlsx.url + '?out=drive'); $('.response-success .d-csv').attr('href', data.csv.url + '?out=drive'); $('.response-success').removeClass('hidden'); @@ -149,38 +153,18 @@ }); }); - /* Click drive xlsx behaviour */ - $('#drive-xlsx').click(function(){ - data - .submit() - .done(function(){ - $('#processing-modal').modal('show'); - $.ajax($('.response-success .d-xlsx').attr('href'), { 'dataType': 'json' }) - .done(function(data){ - $('.google-drive-success .file-google-name').html(data.name); - $('.google-drive-success .file-google-id').html(data.id); - $('.google-drive-success').removeClass('hidden'); - $('.response-fail').addClass('hidden'); - }) - .fail(function(jqXHR, textStatus, errorThrown){ - $('.response-fail').html( - '{% trans "Error saving to Google Drive: " %}' - + ( jqXHR.responseText || textStatus )); - $('.response-fail').removeClass('hidden'); - }) - .always(function(){ - $('#processing-modal').modal('hide'); - }); - }) - }); + $('.f-file').click(function () { + window.location = $(this).attr('href'); + }); - /* Click drive csv behaviour */ - $('#drive-csv').click(function(){ - data + /* Click drive xlsx behaviour */ + $('.d-drive').click(function(){ + url = $(this).attr('href') + data .submit() .done(function(){ $('#processing-modal').modal('show'); - $.ajax($('.response-success .d-csv').attr('href'), { 'dataType': 'json' }) + $.ajax(url, { 'dataType': 'json' }) .done(function(data){ $('.google-drive-success .file-google-name').html(data.name); $('.google-drive-success .file-google-id').html(data.id); From 2771ba49925c8ff9a9cfb51baef0c71898098a37 Mon Sep 17 00:00:00 2001 From: Andres Aguilera Date: Wed, 1 Apr 2020 17:01:02 -0400 Subject: [PATCH 21/26] Fix warning --- default/views.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/default/views.py b/default/views.py index af08f24..ecc3614 100644 --- a/default/views.py +++ b/default/views.py @@ -43,13 +43,11 @@ def retrieve_result(request, folder, id, format=None): file = DataFile(prefix, ext, id=str(id), folder=folder) - if output is None: - return FileResponse(open(file.path, 'rb'), filename=filename, as_attachment=True) - elif output == 'drive': + if output == 'drive': is_test = request.GET.get('test') return upload_to_drive(filename, file.path, format, test=is_test) else: - raise Http404('Invalid output') + return FileResponse(open(file.path, 'rb'), filename=filename, as_attachment=True) def index(request): From ad963179d2f28e44bcc1addf8d53f5decb010902 Mon Sep 17 00:00:00 2001 From: Andres Aguilera Date: Fri, 3 Apr 2020 14:40:55 -0400 Subject: [PATCH 22/26] Add authentication flow tests --- default/drive_options.py | 23 ++++++++++++++--------- tests/test_drive_options.py | 27 ++++++++++++++++++++++++++- 2 files changed, 40 insertions(+), 10 deletions(-) diff --git a/default/drive_options.py b/default/drive_options.py index 1f846df..c32b706 100644 --- a/default/drive_options.py +++ b/default/drive_options.py @@ -1,5 +1,6 @@ from __future__ import print_function from django.http import HttpResponse, JsonResponse +from google.auth.exceptions import DefaultCredentialsError from google.auth.transport.requests import Request from google_auth_oauthlib.flow import InstalledAppFlow from googleapiclient.discovery import build @@ -9,25 +10,29 @@ from ocdstoucan.settings import OCDS_TOUCAN_CREDENTIALS_DRIVE -def upload_to_drive(filename, filepath, format=None, test=False): +SCOPES = 'https://www.googleapis.com/auth/drive.file' + + +def upload_to_drive(filename, filepath, format=None, test=None, credentials=None): try: - credentials = None - SCOPES = 'https://www.googleapis.com/auth/drive.file' - if (not credentials or not credentials.valid) and not test: + if not credentials or not credentials.valid: if credentials and credentials.expired and credentials.refresh_token: credentials.refresh(Request()) else: flow = InstalledAppFlow.from_client_secrets_file( OCDS_TOUCAN_CREDENTIALS_DRIVE, SCOPES) - credentials = flow.run_local_server(port=0) - if test: - service = None - else: - service = build('drive', 'v3', credentials=credentials) + if not test: + credentials = flow.run_local_server(port=0) + if credentials and not credentials.valid: + raise AccessDeniedError + service = build('drive', 'v3', credentials=credentials) except AccessDeniedError: return HttpResponse("Access Denied", status=400) + except DefaultCredentialsError: + service = None + try: if format == 'xlsx': mimeType = 'application/vnd.google-apps.spreadsheet' diff --git a/tests/test_drive_options.py b/tests/test_drive_options.py index 30c2441..206a0ba 100644 --- a/tests/test_drive_options.py +++ b/tests/test_drive_options.py @@ -1,4 +1,8 @@ -from tests import ViewTestCase, ViewTests +from default import drive_options + +from unittest.mock import Mock + +from tests import ViewTestCase, ViewTests, path class DriveTestCase(ViewTestCase, ViewTests): @@ -13,3 +17,24 @@ def test_go_with_files(self): for extension, content in contents.items(): response = self.client.get(content['url'] + '?out=drive&test=true') self.assertEqual(response.status_code, 200) + + def test_access_denied(self): + credentials = Mock(valid=False, expired=False, refresh_token='test') + response = drive_options.upload_to_drive( + filename="test", + filepath="test", + test=True, + credentials=credentials + ) + self.assertEqual(response.status_code, 400) + + def test_fail_uploading(self): + credentials = Mock(valid=False, expired=True, refresh_token='test') + for file in self.files: + response = drive_options.upload_to_drive( + filename="test", + filepath=path(file), + format="json", + credentials=credentials + ) + self.assertEqual(response.status_code, 400) From 554eab1624d6443c983528700b62fe29dea3dd4e Mon Sep 17 00:00:00 2001 From: Andres Aguilera Date: Fri, 3 Apr 2020 15:18:09 -0400 Subject: [PATCH 23/26] Reorder imports --- tests/test_drive_options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_drive_options.py b/tests/test_drive_options.py index 206a0ba..7b25e22 100644 --- a/tests/test_drive_options.py +++ b/tests/test_drive_options.py @@ -2,7 +2,7 @@ from unittest.mock import Mock -from tests import ViewTestCase, ViewTests, path +from tests import path, ViewTestCase, ViewTests class DriveTestCase(ViewTestCase, ViewTests): From 96b148eed0981631ebbed7c8910055f2059771e1 Mon Sep 17 00:00:00 2001 From: Andres Aguilera Date: Fri, 3 Apr 2020 15:18:34 -0400 Subject: [PATCH 24/26] Add credentials.json example --- credentials.json | 1 + 1 file changed, 1 insertion(+) create mode 100644 credentials.json diff --git a/credentials.json b/credentials.json new file mode 100644 index 0000000..eaf42bd --- /dev/null +++ b/credentials.json @@ -0,0 +1 @@ +{"installed":{"client_id":"example.com","project_id":"example","auth_uri":"https://example.com","token_uri":"https://example.com/token","client_secret":"example"}} From 4f9befb598962a559b6df0bdb888154fa0a8d248 Mon Sep 17 00:00:00 2001 From: Andres Aguilera Date: Fri, 3 Apr 2020 15:28:18 -0400 Subject: [PATCH 25/26] Fix warning --- tests/test_drive_options.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_drive_options.py b/tests/test_drive_options.py index 7b25e22..b6b4fc4 100644 --- a/tests/test_drive_options.py +++ b/tests/test_drive_options.py @@ -1,8 +1,8 @@ -from default import drive_options - from unittest.mock import Mock -from tests import path, ViewTestCase, ViewTests +from default import drive_options + +from tests import ViewTestCase, ViewTests, path class DriveTestCase(ViewTestCase, ViewTests): From 1853422d20bd747cac3adbb8a43fea19dc71dd64 Mon Sep 17 00:00:00 2001 From: Andres Aguilera Date: Fri, 3 Apr 2020 15:31:52 -0400 Subject: [PATCH 26/26] Indent credentials.json --- credentials.json | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/credentials.json b/credentials.json index eaf42bd..1ff3dc1 100644 --- a/credentials.json +++ b/credentials.json @@ -1 +1,9 @@ -{"installed":{"client_id":"example.com","project_id":"example","auth_uri":"https://example.com","token_uri":"https://example.com/token","client_secret":"example"}} +{ + "installed": { + "client_id": "example.com", + "project_id": "example", + "auth_uri": "https://example.com", + "token_uri": "https://example.com/token", + "client_secret": "example" + } +}