Merge 11cff4b into c4c6838

knowledge_repo/app/app.py

@@ -172,6 +172,7 @@ def handle_insufficient_permissions(error):
         self.register_blueprint(routes.editor.blueprint)
         self.register_blueprint(routes.groups.blueprint)
         self.register_blueprint(routes.auth.blueprint)
+        self.register_blueprint(routes.polly_api.blueprint)
         KnowledgeAuthProvider.register_auth_provider_blueprints(self)
 
         if self.config['DEBUG']:
@@ -273,6 +274,19 @@ def format_date(date):
             except:
                 return date
 
+    def append_repo(self,name,uri):
+        temp = self.repository
+        self.repository = knowledge_repo.KnowledgeRepository.append_for_uri(name,uri,temp)
+        self.db_update_index(check_timeouts=False, force=True )
+        return self.repository
+
+
+    def append_repo_obj(self,name,obj):
+        temp = self.repository
+        self.repository = knowledge_repo.KnowledgeRepository.append_obj(name,obj,temp)
+        self.db_update_index(check_timeouts=False, force=True)
+        return self.repository
+
     @property
     def repository(self):
         return getattr(self, '_repository')

knowledge_repo/app/auth_providers/oauth2.py

@@ -156,15 +156,17 @@ def extract_from_dict(d, key):
             if isinstance(key, (list, tuple)):
                 if len(key) == 1:
                     key = key[0]
-                else:
-                    return extract_from_dict(d[key[0]], key[1:])
-            if isinstance(key, six.string_types):
+                elif d[key[0]] is None:
+                    return extract_from_dict(d, key[1:])
+            else:
+                return extract_from_dict(d, key[0])
+            if isinstance(key, str):
                 return d[key]
             raise RuntimeError("Invalid key type: {}.".format(key))
 
         response = self.oauth_client.get(self.get_endpoint_url(self.user_info_endpoint), verify=self.verify_ssl_certs)
         try:
-            response_dict = json.loads(response.content)
+            response_dict = json.loads(request.content.decode('utf-8'))
             identifier = extract_from_dict(response_dict, self.user_info_mapping['identifier'])
             if identifier is None:
                 raise ValueError("identifier '{}' not found in authentication response".format(self.user_info_mapping['identifier']))

knowledge_repo/app/routes/__init__.py

@@ -9,3 +9,4 @@
 from . import editor
 from . import groups
 from . import debug
+from . import polly_api 

knowledge_repo/app/routes/index.py

@@ -51,6 +51,17 @@ def render_index():
     return redirect('/feed')
 
 
+@blueprint.route('/testupload')
+@PageView.logged
+def test_upload():
+    global current_repo,current_app
+    #repo = current_app.append_repo("3","kr-test")
+    #current_repo = repo
+    dbobj = current_repo.migrate_to_dbrepo("ucla_rnaseq_tcga_analysis","ucla-kr")
+
+    current_repo = current_app.append_repo_obj("3",dbobj)
+    return redirect('/feed')    
+
 @blueprint.route('/favorites')
 @PageView.logged
 @login_required
@@ -82,6 +93,7 @@ def render_favorites():
 @permissions.index_view.require()
 def render_feed():
     """ Renders the index-feed view """
+    global current_repo,current_app
     feed_params = from_request_get_feed_params(request)
     posts, post_stats = get_posts(feed_params)
     for post in posts:

knowledge_repo/app/routes/polly_api.py

@@ -0,0 +1,41 @@
+"""
+APIs for polly and their routes
+Includes:
+    - /api/uploadkr
+    - /api/uploadpost
+
+"""
+
+import os
+import json
+from builtins import str
+from collections import namedtuple
+from flask import request, render_template, redirect, Blueprint, current_app, make_response
+from flask_login import login_required
+from sqlalchemy import case, desc
+
+from .. import permissions
+from ..proxies import db_session, current_repo
+from ..utils.posts import get_posts
+from ..models import Post, Tag, User, PageView
+from ..utils.requests import from_request_get_feed_params
+from ..utils.render import render_post_tldr
+from ..utils.s3_talk import download_dir,download_from_s3  
+blueprint = Blueprint('api', __name__, template_folder='../templates', static_folder='../static')
+
+@blueprint.route('/api/uploadkr')
+@PageView.logged
+def upload_kr():
+    """
+    API to upload a KR to the server
+    args:
+        S3-Path 
+    """
+    global current_repo,current_app
+    path = request.args.get('path')
+    dir_name,dir_path = download_dir(path)
+    dbobj  = current_repo.migrate_to_dbrepo(dir_path,dir_name)
+    current_repo = current_app.append_repo_obj(dir_name,dbobj)
+    return redirect('/feed')
+
+

knowledge_repo/app/utils/s3_talk.py

@@ -0,0 +1,70 @@
+"""
+Given S3 path download to local and trigger upload to database repository
+"""
+import os
+import boto3,botocore
+import tempfile
+import errno
+
+BUCKET_NAME = os.environ['KR_BUCKET_NAME']
+S3 = boto3.resource('s3')
+CLIENT = boto3.client('s3')
+
+def assert_dir(path):
+    """
+    Checks if directory tree in path exists.     :param path: the path to check if it exists
+    """
+    try:
+        os.makedirs(path)
+    except OSError as e:
+        if e.errno != errno.EEXIST:
+            raise
+
+def _extract_name(path):
+# Internal function to return name of the folder/file to use downstream
+    if path.endswith('/'):
+        return path.split('/')[-2]
+    else:
+        return path.split('/')[-1]
+
+def download_dir(path):
+    """
+    Downloads recursively the given S3 path to the target directory.
+    arg path: The S3 directory to download.
+    """
+
+    # Handle missing / at end of prefix
+    if not path.endswith('/'):
+        path += '/'
+
+    dir_name = _extract_name(path)
+    target = "/tmp/%s/"%dir_name
+
+    paginator = CLIENT.get_paginator('list_objects_v2')
+    for result in paginator.paginate(Bucket=BUCKET_NAME, Prefix=path):
+        # Download each file individually
+        for key in result['Contents']:
+            # Calculate relative path
+            rel_path = key['Key'][len(path):]
+            # Skip paths ending in /
+            if not key['Key'].endswith('/'):
+                local_file_path =  os.path.join(target, rel_path)
+                # Make sure directories exist
+                local_file_dir = os.path.dirname(local_file_path)
+                assert_dir(local_file_dir)
+                CLIENT.download_file(BUCKET_NAME, key['Key'], local_file_path)
+    return dir_name,target
+
+
+
+
+def download_from_s3(path):
+    try: 
+        fname = _extract_path(path)
+        temp_name = '/tmp/%s'%fname
+        S3.Bucket(BUCKET_NAME).download_file(path,temp_name)
+    except botocore.exceptions.ClientError as e:
+        if e.response['Error']['Code'] == "404":
+            return -1
+
+    return fname,temp_name

knowledge_repo/repositories/dbrepository.py

@@ -45,7 +45,7 @@ def init(self, auto_create=True):
                               Column('revision', Integer, default=0),
                               Column('status', Integer, default=self.PostStatus.DRAFT.value),
                               Column('ref', String(512)),
-                              Column('data', LargeBinary))
+                              Column('data', LargeBinary(length=(2**32)-1))) # Increased length to support large notebooks
         self.engine = create_engine(engine_uri, pool_recycle=3600)
         self.session = scoped_session(sessionmaker(bind=self.engine))
         if auto_create:

knowledge_repo/repository.py

@@ -58,6 +58,42 @@ def for_uris(cls, uri):
         krs = {name: cls.for_uri(uri) for name, uri in list(uris.items())}
         return MetaKnowledgeRepository(krs)
 
+    @classmethod
+    def migrate_to_dbrepo(cls,gitpath,newpath):
+        # A new (supposedly) git repository is going to be uploaded
+        # That must be migrated to the database. 
+        # Inputs : 
+        # gitpath
+        # newpath
+
+        from .repositories.dbrepository import DbKnowledgeRepository
+        newpath = "mysql://abhi:1234@localhost/knowledgerepo:%s"%newpath
+        db_obj = DbKnowledgeRepository(newpath)
+        gitkr = cls.for_uri(gitpath)
+        for post in gitkr.posts():
+            new_post = db_obj.add(post)
+            new_post_status = new_post.status
+            db_obj.submit(new_post.path)
+            db_obj.accept(new_post.path)
+            db_obj.publish(new_post.path)
+        return db_obj
+
+    @classmethod
+    def append_obj(cls,name,new_obj,meta_repo):
+        from .repositories.meta import MetaKnowledgeRepository
+        krs = meta_repo.uri
+        krs[name] = new_obj
+        meta_repo = MetaKnowledgeRepository(krs)
+        return meta_repo
+
+    @classmethod
+    def append_for_uri(cls,name,uri,meta_repo):
+        from .repositories.meta import MetaKnowledgeRepository
+        krs = meta_repo.uri
+        krs[name] = cls.for_uri(uri)
+        meta_repo = MetaKnowledgeRepository(krs)
+        return meta_repo
+
     @classmethod
     def from_uri(cls, url, *args, **kwargs):
         return cls(url, *args, **kwargs)

scripts/knowledge_repo

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 from __future__ import print_function
 from __future__ import unicode_literals
 
@@ -42,6 +42,13 @@ if os.path.exists(os.path.join(contrib_dir, '__init__.py')):
 # we finish constructing the entire parser so that the syntax and arguments can change
 # from version to version of this script.
 
+# Repeating what the argparse already does
+def parse_repo_string(repo_str):
+    prefix_pattern = re.compile('^(?:\{(?P<name>[a-zA-Z_0-9]*)\})?(?P<uri>.*)$')
+
+    prefix = prefix_pattern.match(repo_str)
+    return prefix.groups()
+
 class ParseRepositories(argparse.Action):
 
     def __init__(self, **kwargs):