Merge branch 'master' into 14-send_results

# Conflicts:
#	default/static/css/custom.css
#	default/static/js/uploader.js
#	default/templates/default/base-uploader.html
#	default/templates/default/to-json.html
#	default/templates/default/to-spreadsheet.html
#	default/views.py
#	locale/es/LC_MESSAGES/django.po

default/data_file.py

@@ -50,7 +50,7 @@ def path(self):
         """
         Returns the full path to the file.
         """
-        return os.path.join(self._directory, self._name)
+        return os.path.join(self._directory, self.name)
 
     @property
     def url(self):
@@ -73,16 +73,19 @@ def json(self, **kwargs):
         with open(self.path, encoding='utf-8') as f:
             return json.load(f, **kwargs)
 
-    def write(self, file):
+    def write(self, content):
         """
         Write another file's contents to this file.
 
-        :param file: a ``django.core.files.File`` object
+        :param content: either a ``django.core.files.File`` object or a bytes array
         """
         self._makedirs()
         with open(self.path, 'wb') as f:
-            for chunk in file.chunks():
-                f.write(chunk)
+            if getattr(content, 'chunks', None):
+                for chunk in content.chunks():
+                    f.write(chunk)
+            else:
+                f.write(content)
 
     def write_json_to_zip(self, files):
         """
@@ -99,7 +102,7 @@ def write_json_to_zip(self, files):
                 zipfile.writestr(name, json_dumps(content) + '\n')
 
     @property
-    def _name(self):
+    def name(self):
         """
         Returns the file's name.
         """

default/decorators.py

@@ -71,3 +71,14 @@ def wrap(request, *args, **kwargs):
         return function(request, *args, **kwargs)
 
     return wrap
+
+
+def clear_drive_session_vars(function):
+    @wraps(function)
+    def wrap(request, *args, **kwargs):
+        val = function(request, *args, **kwargs)
+        if 'auth_status' in request.session and request.session['auth_status'] in ('completed', 'success', 'failed'):
+            for key in ('auth_status', 'auth_response', 'auth_status_error', 'google_drive_file'):
+                request.session.pop(key, None)
+        return val
+    return wrap

default/google_drive.py

@@ -0,0 +1,121 @@
+import logging
+from datetime import datetime
+
+from django.http import JsonResponse
+from django.utils.translation import gettext_lazy as _
+from google.auth.transport.requests import Request as GoogleRequest
+from google.oauth2.credentials import Credentials
+from google_auth_oauthlib.flow import Flow
+from googleapiclient.discovery import build
+from googleapiclient.http import MediaFileUpload
+
+from default.data_file import DataFile
+from ocdstoucan.settings import OCDS_TOUCAN_GOOGLE_API_CREDENTIALS_FILE
+
+"""
+Code based in the documentation here: https://developers.google.com/identity/protocols/oauth2/web-server
+
+See also:
+
+* Oauth2 protocol: https://developers.google.com/identity/protocols/oauth2/
+* Credentials class docs: https://google-auth.readthedocs.io/en/latest/reference/google.oauth2.credentials.html
+* Flow class docs: https://google-auth-oauthlib.readthedocs.io/en/latest/reference/google_auth_oauthlib.flow.html
+
+"""
+
+logger = logging.getLogger(__name__)
+
+SCOPES = ['https://www.googleapis.com/auth/drive.file']
+
+google_api_messages = {
+    '_default': _('There was an issue when trying to upload the file to Google Drive, please try again later.'),
+    'access_denied': _(
+        'There was an authorization issue when saving the file to Google Drive, please try again later.')
+}
+
+mime_types = {
+    '_default': 'application/zip',
+    '.csv': 'text/csv',
+    '.xlsx': 'application/vnd.google-apps.spreadsheet'
+}
+
+
+def upload_to_drive(request):
+
+    datafile = DataFile(**request.session['google_drive_file'])
+    credentials = get_credentials_from_session(request)
+
+    try:
+        if credentials:
+            if credentials.expired:
+                credentials.refresh(GoogleRequest())
+                save_refresh_info(request, credentials)
+        else:
+            flow = Flow.from_client_secrets_file(OCDS_TOUCAN_GOOGLE_API_CREDENTIALS_FILE, SCOPES)
+            flow.redirect_uri = request.build_absolute_uri('googleapi-auth-response')
+
+            authorization_response = request.session['auth_response']
+            flow.fetch_token(authorization_response=authorization_response)
+
+            credentials = flow.credentials
+            save_credentials_to_session(request, credentials)
+
+        service = build('drive', 'v3', credentials=credentials)
+
+        if datafile.ext in mime_types.keys():
+            mime_type = mime_types[datafile.ext]
+        else:
+            mime_type = mime_types['_default']
+
+        file_metadata = {
+            'name': datafile.name,
+            'mimeType': mime_type
+        }
+        media = MediaFileUpload(datafile.path, mimetype=mime_type, resumable=True)
+        results = service.files().create(body=file_metadata, media_body=media, fields='id').execute()
+
+        return JsonResponse({
+            'authenticated': True,
+            'status': 'success',
+            'name': datafile.name,
+            'url': "https://drive.google.com/file/d/" + results["id"]
+        })
+
+    except Exception:
+        return JsonResponse({
+            'status': 'failed',
+            'message': google_api_messages['_default']
+        }, status=500)
+
+
+def get_credentials_from_session(request):
+    if 'credentials' in request.session:
+        info = request.session['credentials']
+        credentials = Credentials(info['token'], refresh_token=info['refresh_token'],
+                                  id_token=info['id_token'], token_uri=info['token_uri'],
+                                  client_id=info['client_id'], client_secret=info['client_secret'],
+                                  scopes=info['scopes'])
+
+        credentials.expiry = datetime.utcfromtimestamp(info['expiry'])
+
+        return credentials
+    return None
+
+
+def save_credentials_to_session(request, credentials):
+    request.session['credentials'] = {
+        'token': credentials.token,
+        'refresh_token': credentials.refresh_token,
+        'id_token': credentials.id_token,
+        'token_uri': credentials.token_uri,
+        'client_id': credentials.client_id,
+        'client_secret': credentials.client_secret,
+        'scopes': credentials.scopes,
+        'expiry': credentials.expiry.timestamp()
+    }
+    request.session.modified = True
+
+
+def save_refresh_info(request, credentials):
+    request.session['credentials']['token'] = credentials.token
+    request.session['credentials']['expiry'] = credentials.expiry.timestamp()

default/mapping_sheet.py

@@ -1,32 +1,43 @@
 from io import StringIO
 
 import jsonref
-from django.http.response import HttpResponse
+from django.http import HttpResponse
 from ocdsextensionregistry import ProfileBuilder
 from ocdskit.mapping_sheet import mapping_sheet as mapping_sheet_method
 
+from default.data_file import DataFile
 
-def get_mapping_sheet_from_url(url):
+
+def get_mapping_sheet_from_url(url, as_response=False):
     schema = jsonref.load_uri(url)
-    return _get_mapping_sheet(schema)
+    return _get_mapping_sheet(schema, as_response)
 
 
 def get_mapping_sheet_from_uploaded_file(uploaded):
     schema = jsonref.load(uploaded)
     return _get_mapping_sheet(schema)
 
 
-def get_extended_mapping_sheet(extensions, version):
+def get_extended_mapping_sheet(extensions, version, as_response=False):
     builder = ProfileBuilder(version, extensions)
     schema = jsonref.JsonRef.replace_refs(builder.patched_release_schema())
-    return _get_mapping_sheet(schema)
+    return _get_mapping_sheet(schema, as_response)
 
 
-def _get_mapping_sheet(data):
+def _get_mapping_sheet(data, as_response=False):
     io = StringIO()
     mapping_sheet_method(data, io, infer_required=True)
 
-    response = HttpResponse(io.getvalue(), content_type='text/csv')
-    response['Content-Disposition'] = 'attachment; filename="mapping-sheet.csv"'
+    if as_response:
+        response = HttpResponse(io.getvalue(), content_type='text/csv')
+        response['Content-Disposition'] = 'attachment; filename="result.csv"'
+        return response
+
+    data_file = DataFile('result', '.csv')
+    data_file.write(io.getvalue().encode('utf-8'))
 
-    return response
+    return {
+        'url': data_file.url + 'csv/',
+        'driveUrl': data_file.url.replace('result', 'google-drive-save-start') + 'csv/',
+        'size': data_file.size
+    }

default/static/css/custom.css

@@ -40,13 +40,14 @@
 {
     text-decoration: none;
 }
-.info.panel .panel-body:before{
+.info.panel .panel-heading:before{
     content: '\e085';
     font-family: "Glyphicons Halflings";
     font-size: 2em;
     float: left;
     margin-right: 10px;
-    color: #9BAF00; 
+    margin-top: -8px;
+    color: #9BAF00;
 }
 .schema-nav {
     margin-bottom: 10px;
@@ -76,10 +77,24 @@
     position: relative;
     min-height: 100vh;
 }
+
 .content-wrap.ocp-content {
     padding-bottom: 12rem;
 }
 .form-send {
     margin-bottom: 0;
     margin-top: 10px;
 }
+
+.panel-title a {
+    color: #9baf00;
+    text-decoration: none;
+}
+
+.panel-title a .control {
+    color: #8c8c8c;
+}
+
+.panel-title a .control .glyphicon{
+    top: 5px;
+}

default/static/js/base.js

@@ -0,0 +1,38 @@
+var toucanApp = {};
+(function(){
+    function infoControl() {
+        $('.info #collapse-panel').collapse('toggle');
+    }
+
+    function whenPanelShows(e) {
+        $('#collapse-control .control.more').addClass('hidden');
+        $('#collapse-control .control.less').removeClass('hidden');
+    }
+
+    function whenPanelHides(e) {
+        $('#collapse-control .control.less').addClass('hidden');
+        $('#collapse-control .control.more').removeClass('hidden');
+    }
+
+    this.showProcessingModal = function(customSelector) {
+        if (customSelector) {
+            $('#processing-modal ' + customSelector).removeClass('hidden');
+            $('#processing-modal .default-message').addClass('hidden');
+        }
+        $('#processing-modal').modal('show');
+    };
+
+    this.hideProcessingModal = function(customSelector) {
+        if (customSelector) {
+            $('#processing-modal ' + customSelector).addClass('hidden');
+            $('#processing-modal .default-message').removeClass('hidden');
+        }
+        $('#processing-modal').modal('hide');
+    };
+
+    // add handler
+    $('#collapse-control').click(infoControl);
+    $('.info #collapse-panel')
+        .on('show.bs.collapse', whenPanelShows)
+        .on('hide.bs.collapse', whenPanelHides)
+}).apply(toucanApp);

default/static/js/googleDriveUploader.js

@@ -0,0 +1,81 @@
+(function(){
+    var app = this;
+    var _googleApiURL = 'https://accounts.google.com/o/oauth2/v2/auth';
+    var _myBaseUrl = window.location.href.match('(https?:\/\/[^/]*\/)')[0];
+    var _googleApiParams = {
+        redirect_uri: _myBaseUrl + 'googleapi-auth-response',
+        access_type: 'offline',
+        response_type: 'code',
+        scope: 'https://www.googleapis.com/auth/drive.file'
+    };
+
+    this.setGoogleApiClientID = function(id) {
+      _googleApiParams['client_id'] = id;
+    };
+
+    function doPoll(successCallback, errorCallback){
+        $.get(_myBaseUrl + 'google-drive-save-status')
+            .done(function (data) {
+                if (data.status === 'waiting'){
+                    doPoll(successCallback, errorCallback);
+                }
+                else if (data.status === 'success') {
+                    successCallback(data);
+                }
+            })
+            .fail(function(jqXHR, textStatus, errorThrown) {
+                errorCallback(jqXHR, textStatus, errorThrown);
+            });
+    }
+
+    function saveToDrive(event) {
+
+        var successCallback = function(data, classSelector){
+            var linkSelector = '.response-success .open-drive-link';
+            var saveSelector = '.response-success .file.save-drive-link';
+            if (classSelector){
+                linkSelector = linkSelector + '.' + classSelector;
+                saveSelector = saveSelector + '.' + classSelector;
+            }
+
+            $(linkSelector + ' a').attr('href', data.url);
+            $(linkSelector).removeClass('hidden');
+            $(saveSelector).addClass('hidden');
+            $('.response-fail-empty').addClass('hidden');
+            app.hideProcessingModal('.auth-message');
+        };
+
+        var failedCallback = function(jqXHR, textStatus){
+            var jsonResponse = $.parseJSON(jqXHR.responseText);
+            $('.response-fail-empty .message-content').html(
+                ( jsonResponse.message || jqXHR.responseText || textStatus )
+            );
+            $('.response-fail-empty').removeClass('hidden');
+            app.hideProcessingModal('.auth-message');
+        };
+
+        var url = $(event.target).attr('data-url');
+        var classSelector = $(event.target).attr('data-class');
+
+        app.showProcessingModal('.auth-message');
+        $.ajax(url)
+            .done(function(response)
+            {
+                // there are credentials already saved for this user, so the response contains the details of the uploaded file
+                if (response.authenticated && response.authenticated === true)
+                {
+                    successCallback(response, classSelector);
+                }
+                else
+                {
+                    // open the authentication page in a new window
+                    window.open(_googleApiURL + '?' + $.param(_googleApiParams), '_blank');
+                    // poll and wait for a response
+                    doPoll(successCallback, failedCallback)
+                }
+            })
+            .fail(failedCallback);
+    }
+    /* click save to Drive button behaviour */
+    $('.file.save-drive-link').click(saveToDrive);
+}).apply(toucanApp);