This repository has been archived by the owner on Dec 13, 2022. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 5
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add output to Google Drive option (#67)
* Add output to google drive option * Add upload to google drive options * Add translations * Fix imports * Fix warning * Add translations * Add module google * Fix large line * Update requeriments * Update translations * Change URL format and methods * Add mime types and JsonResponse * Add drive button and new functions * Update fails * Change saveDrive * Add new translations * Add tests * Update tests * Change words in translation * Order code and change 'Download' to a button * Fix warning * Add authentication flow tests * Reorder imports * Add credentials.json example * Fix warning * Indent credentials.json * [WIP] Increase coverage for drive_options.py * Update exceptions * Update messages * Update translations * Change tags * Update messages and exceptions * Update templates * Update translations * Update tests * Add Exception * Update drive button * Update translations * Update tests * Update Drive button * Update tags * Update translations * Correct saveDrive function * Update changes * Delete google package * Save to Drive, and code refactors * Fix quality issues * Fix sorting issues * Read secrets file outside settings.py * Remove sslserver and old credentials.json * Minor fixes after review * Fix quality issues * Fix test issue Co-authored-by: Romina Fernandez <romifz@gmail.com>
- Loading branch information
1 parent
fe9bb37
commit dba699e
Showing
38 changed files
with
1,476 additions
and
803 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,121 @@ | ||
import logging | ||
from datetime import datetime | ||
|
||
from django.http import JsonResponse | ||
from django.utils.translation import gettext_lazy as _ | ||
from google.auth.transport.requests import Request as GoogleRequest | ||
from google.oauth2.credentials import Credentials | ||
from google_auth_oauthlib.flow import Flow | ||
from googleapiclient.discovery import build | ||
from googleapiclient.http import MediaFileUpload | ||
|
||
from default.data_file import DataFile | ||
from ocdstoucan.settings import OCDS_TOUCAN_GOOGLE_API_CREDENTIALS_FILE | ||
|
||
""" | ||
Code based in the documentation here: https://developers.google.com/identity/protocols/oauth2/web-server | ||
See also: | ||
* Oauth2 protocol: https://developers.google.com/identity/protocols/oauth2/ | ||
* Credentials class docs: https://google-auth.readthedocs.io/en/latest/reference/google.oauth2.credentials.html | ||
* Flow class docs: https://google-auth-oauthlib.readthedocs.io/en/latest/reference/google_auth_oauthlib.flow.html | ||
""" | ||
|
||
logger = logging.getLogger(__name__) | ||
|
||
SCOPES = ['https://www.googleapis.com/auth/drive.file'] | ||
|
||
google_api_messages = { | ||
'_default': _('There was an issue when trying to upload the file to Google Drive, please try again later.'), | ||
'access_denied': _( | ||
'There was an authorization issue when saving the file to Google Drive, please try again later.') | ||
} | ||
|
||
mime_types = { | ||
'_default': 'application/zip', | ||
'.csv': 'text/csv', | ||
'.xlsx': 'application/vnd.google-apps.spreadsheet' | ||
} | ||
|
||
|
||
def upload_to_drive(request): | ||
|
||
datafile = DataFile(**request.session['google_drive_file']) | ||
credentials = get_credentials_from_session(request) | ||
|
||
try: | ||
if credentials: | ||
if credentials.expired: | ||
credentials.refresh(GoogleRequest()) | ||
save_refresh_info(request, credentials) | ||
else: | ||
flow = Flow.from_client_secrets_file(OCDS_TOUCAN_GOOGLE_API_CREDENTIALS_FILE, SCOPES) | ||
flow.redirect_uri = request.build_absolute_uri('googleapi-auth-response') | ||
|
||
authorization_response = request.session['auth_response'] | ||
flow.fetch_token(authorization_response=authorization_response) | ||
|
||
credentials = flow.credentials | ||
save_credentials_to_session(request, credentials) | ||
|
||
service = build('drive', 'v3', credentials=credentials) | ||
|
||
if datafile.ext in mime_types.keys(): | ||
mime_type = mime_types[datafile.ext] | ||
else: | ||
mime_type = mime_types['_default'] | ||
|
||
file_metadata = { | ||
'name': datafile.name, | ||
'mimeType': mime_type | ||
} | ||
media = MediaFileUpload(datafile.path, mimetype=mime_type, resumable=True) | ||
results = service.files().create(body=file_metadata, media_body=media, fields='id').execute() | ||
|
||
return JsonResponse({ | ||
'authenticated': True, | ||
'status': 'success', | ||
'name': datafile.name, | ||
'url': "https://drive.google.com/file/d/" + results["id"] | ||
}) | ||
|
||
except Exception: | ||
return JsonResponse({ | ||
'status': 'failed', | ||
'message': google_api_messages['_default'] | ||
}, status=500) | ||
|
||
|
||
def get_credentials_from_session(request): | ||
if 'credentials' in request.session: | ||
info = request.session['credentials'] | ||
credentials = Credentials(info['token'], refresh_token=info['refresh_token'], | ||
id_token=info['id_token'], token_uri=info['token_uri'], | ||
client_id=info['client_id'], client_secret=info['client_secret'], | ||
scopes=info['scopes']) | ||
|
||
credentials.expiry = datetime.utcfromtimestamp(info['expiry']) | ||
|
||
return credentials | ||
return None | ||
|
||
|
||
def save_credentials_to_session(request, credentials): | ||
request.session['credentials'] = { | ||
'token': credentials.token, | ||
'refresh_token': credentials.refresh_token, | ||
'id_token': credentials.id_token, | ||
'token_uri': credentials.token_uri, | ||
'client_id': credentials.client_id, | ||
'client_secret': credentials.client_secret, | ||
'scopes': credentials.scopes, | ||
'expiry': credentials.expiry.timestamp() | ||
} | ||
request.session.modified = True | ||
|
||
|
||
def save_refresh_info(request, credentials): | ||
request.session['credentials']['token'] = credentials.token | ||
request.session['credentials']['expiry'] = credentials.expiry.timestamp() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,32 +1,43 @@ | ||
from io import StringIO | ||
|
||
import jsonref | ||
from django.http.response import HttpResponse | ||
from django.http import HttpResponse | ||
from ocdsextensionregistry import ProfileBuilder | ||
from ocdskit.mapping_sheet import mapping_sheet as mapping_sheet_method | ||
|
||
from default.data_file import DataFile | ||
|
||
def get_mapping_sheet_from_url(url): | ||
|
||
def get_mapping_sheet_from_url(url, as_response=False): | ||
schema = jsonref.load_uri(url) | ||
return _get_mapping_sheet(schema) | ||
return _get_mapping_sheet(schema, as_response) | ||
|
||
|
||
def get_mapping_sheet_from_uploaded_file(uploaded): | ||
schema = jsonref.load(uploaded) | ||
return _get_mapping_sheet(schema) | ||
|
||
|
||
def get_extended_mapping_sheet(extensions, version): | ||
def get_extended_mapping_sheet(extensions, version, as_response=False): | ||
builder = ProfileBuilder(version, extensions) | ||
schema = jsonref.JsonRef.replace_refs(builder.patched_release_schema()) | ||
return _get_mapping_sheet(schema) | ||
return _get_mapping_sheet(schema, as_response) | ||
|
||
|
||
def _get_mapping_sheet(data): | ||
def _get_mapping_sheet(data, as_response=False): | ||
io = StringIO() | ||
mapping_sheet_method(data, io, infer_required=True) | ||
|
||
response = HttpResponse(io.getvalue(), content_type='text/csv') | ||
response['Content-Disposition'] = 'attachment; filename="mapping-sheet.csv"' | ||
if as_response: | ||
response = HttpResponse(io.getvalue(), content_type='text/csv') | ||
response['Content-Disposition'] = 'attachment; filename="result.csv"' | ||
return response | ||
|
||
data_file = DataFile('result', '.csv') | ||
data_file.write(io.getvalue().encode('utf-8')) | ||
|
||
return response | ||
return { | ||
'url': data_file.url + 'csv/', | ||
'driveUrl': data_file.url.replace('result', 'google-drive-save-start') + 'csv/', | ||
'size': data_file.size | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,38 @@ | ||
var toucanApp = {}; | ||
(function(){ | ||
function infoControl() { | ||
$('.info #collapse-panel').collapse('toggle'); | ||
} | ||
|
||
function whenPanelShows(e) { | ||
$('#collapse-control .control.more').addClass('hidden'); | ||
$('#collapse-control .control.less').removeClass('hidden'); | ||
} | ||
|
||
function whenPanelHides(e) { | ||
$('#collapse-control .control.less').addClass('hidden'); | ||
$('#collapse-control .control.more').removeClass('hidden'); | ||
} | ||
|
||
this.showProcessingModal = function(customSelector) { | ||
if (customSelector) { | ||
$('#processing-modal ' + customSelector).removeClass('hidden'); | ||
$('#processing-modal .default-message').addClass('hidden'); | ||
} | ||
$('#processing-modal').modal('show'); | ||
}; | ||
|
||
this.hideProcessingModal = function(customSelector) { | ||
if (customSelector) { | ||
$('#processing-modal ' + customSelector).addClass('hidden'); | ||
$('#processing-modal .default-message').removeClass('hidden'); | ||
} | ||
$('#processing-modal').modal('hide'); | ||
}; | ||
|
||
// add handler | ||
$('#collapse-control').click(infoControl); | ||
$('.info #collapse-panel') | ||
.on('show.bs.collapse', whenPanelShows) | ||
.on('hide.bs.collapse', whenPanelHides) | ||
}).apply(toucanApp); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,81 @@ | ||
(function(){ | ||
var app = this; | ||
var _googleApiURL = 'https://accounts.google.com/o/oauth2/v2/auth'; | ||
var _myBaseUrl = window.location.href.match('(https?:\/\/[^/]*\/)')[0]; | ||
var _googleApiParams = { | ||
redirect_uri: _myBaseUrl + 'googleapi-auth-response', | ||
access_type: 'offline', | ||
response_type: 'code', | ||
scope: 'https://www.googleapis.com/auth/drive.file' | ||
}; | ||
|
||
this.setGoogleApiClientID = function(id) { | ||
_googleApiParams['client_id'] = id; | ||
}; | ||
|
||
function doPoll(successCallback, errorCallback){ | ||
$.get(_myBaseUrl + 'google-drive-save-status') | ||
.done(function (data) { | ||
if (data.status === 'waiting'){ | ||
doPoll(successCallback, errorCallback); | ||
} | ||
else if (data.status === 'success') { | ||
successCallback(data); | ||
} | ||
}) | ||
.fail(function(jqXHR, textStatus, errorThrown) { | ||
errorCallback(jqXHR, textStatus, errorThrown); | ||
}); | ||
} | ||
|
||
function saveToDrive(event) { | ||
|
||
var successCallback = function(data, classSelector){ | ||
var linkSelector = '.response-success .open-drive-link'; | ||
var saveSelector = '.response-success .file.save-drive-link'; | ||
if (classSelector){ | ||
linkSelector = linkSelector + '.' + classSelector; | ||
saveSelector = saveSelector + '.' + classSelector; | ||
} | ||
|
||
$(linkSelector + ' a').attr('href', data.url); | ||
$(linkSelector).removeClass('hidden'); | ||
$(saveSelector).addClass('hidden'); | ||
$('.response-fail-empty').addClass('hidden'); | ||
app.hideProcessingModal('.auth-message'); | ||
}; | ||
|
||
var failedCallback = function(jqXHR, textStatus){ | ||
var jsonResponse = $.parseJSON(jqXHR.responseText); | ||
$('.response-fail-empty .message-content').html( | ||
( jsonResponse.message || jqXHR.responseText || textStatus ) | ||
); | ||
$('.response-fail-empty').removeClass('hidden'); | ||
app.hideProcessingModal('.auth-message'); | ||
}; | ||
|
||
var url = $(event.target).attr('data-url'); | ||
var classSelector = $(event.target).attr('data-class'); | ||
|
||
app.showProcessingModal('.auth-message'); | ||
$.ajax(url) | ||
.done(function(response) | ||
{ | ||
// there are credentials already saved for this user, so the response contains the details of the uploaded file | ||
if (response.authenticated && response.authenticated === true) | ||
{ | ||
successCallback(response, classSelector); | ||
} | ||
else | ||
{ | ||
// open the authentication page in a new window | ||
window.open(_googleApiURL + '?' + $.param(_googleApiParams), '_blank'); | ||
// poll and wait for a response | ||
doPoll(successCallback, failedCallback) | ||
} | ||
}) | ||
.fail(failedCallback); | ||
} | ||
/* click save to Drive button behaviour */ | ||
$('.file.save-drive-link').click(saveToDrive); | ||
}).apply(toucanApp); |
Oops, something went wrong.