[importer] Add default_s3_home and initiate value in getfilesystem (#…

…1983)

apps/filebrowser/src/filebrowser/views.py

@@ -44,7 +44,7 @@
 from django.utils.http import http_date
 from django.utils.html import escape
 
-from aws.s3.s3fs import S3FileSystemException, S3ListAllBucketsException
+from aws.s3.s3fs import S3FileSystemException, S3ListAllBucketsException, get_s3_home_directory
 from desktop import appmanager
 from desktop.auth.backend import is_admin
 from desktop.lib import i18n
@@ -218,6 +218,14 @@ def view(request, path):
           '/filebrowser/view=' + urllib_quote(home_dir_path.encode('utf-8'), safe=SAFE_CHARACTERS_URI_COMPONENTS)
       )
 
+  if 'default_s3_home' in request.GET:
+    home_dir_path = get_s3_home_directory()
+    if request.fs.isdir(home_dir_path):
+      return format_preserving_redirect(
+          request,
+          '/filebrowser/view=' + urllib_quote(home_dir_path.encode('utf-8'), safe=SAFE_CHARACTERS_URI_COMPONENTS)
+      )
+
   # default_to_home is set in jquery.filechooser.js
   if 'default_to_home' in request.GET:
     home_dir_path = request.user.get_home_directory()

apps/filebrowser/src/filebrowser/views_test.py

@@ -39,7 +39,7 @@
 from aws.s3.s3fs import S3FileSystemException
 from aws.s3.s3test_utils import get_test_bucket
 
-from azure.conf import is_abfs_enabled, is_adls_enabled
+from azure.conf import is_abfs_enabled, is_adls_enabled, ABFS_CLUSTERS
 from django.urls import reverse
 from django.utils.encoding import smart_str
 from nose.plugins.attrib import attr
@@ -56,7 +56,8 @@
 from hadoop.fs.webhdfs import WebHdfs
 from useradmin.models import User, Group
 
-from filebrowser.conf import ENABLE_EXTRACT_UPLOADED_ARCHIVE, MAX_SNAPPY_DECOMPRESSION_SIZE
+from filebrowser.conf import ENABLE_EXTRACT_UPLOADED_ARCHIVE, MAX_SNAPPY_DECOMPRESSION_SIZE,\
+  REMOTE_STORAGE_HOME
 from filebrowser.lib.rwx import expand_mode
 from filebrowser.views import snappy_installed
 
@@ -1467,3 +1468,54 @@ def test_has_default_permissions(self):
       assert_equal(200, response.status_code)
     finally:
       remove_from_group(self.user.username, 'has_adls')
+
+class TestFileChooserRedirect(object):
+
+  def setUp(self):
+    self.client = make_logged_in_client(username="test", groupname="default", recreate=True, is_superuser=False)
+    grant_access('test', 'test', 'filebrowser')
+    add_to_group('test')
+
+    self.user = User.objects.get(username="test")
+
+  def test_hdfs_redirect(self):
+    with patch('desktop.lib.fs.proxyfs.ProxyFS.isdir') as is_dir:
+      is_dir.return_value = True
+
+      # HDFS - default_to_home
+      response = self.client.get('/filebrowser/view=%2F?default_to_home')
+      LOG.info("Response: %s" % response.status_code)
+      assert_equal(302, response.status_code)
+      assert_equal('/filebrowser/view=%2Fuser%2Ftest', response.url)
+
+      # ABFS - default_abfs_home
+      response = self.client.get('/filebrowser/view=%2F?default_abfs_home')
+      LOG.info("Response: %s" % response.status_code)
+      assert_equal(302, response.status_code)
+      assert_equal('/filebrowser/view=abfs%3A%2F%2F', response.url)
+
+      reset = ABFS_CLUSTERS['default'].FS_DEFAULTFS.set_for_testing(
+                  'abfs://data-container@mystorage.dfs.core.windows.net'
+              )
+      try:
+        response = self.client.get('/filebrowser/view=%2F?default_abfs_home')
+        LOG.info("Response: %s" % response.status_code)
+        assert_equal(302, response.status_code)
+        assert_equal('/filebrowser/view=abfs%3A%2F%2Fdata-container', response.url)
+      finally:
+        reset()
+
+      # S3A - default_s3_home
+      response = self.client.get('/filebrowser/view=%2F?default_s3_home')
+      LOG.info("Response: %s" % response.status_code)
+      assert_equal(302, response.status_code)
+      assert_equal('/filebrowser/view=s3a%3A%2F%2F', response.url)
+
+      reset = REMOTE_STORAGE_HOME.set_for_testing('s3a://my_bucket')
+      try:
+        response = self.client.get('/filebrowser/view=%2F?default_s3_home')
+        LOG.info("Response: %s" % response.status_code)
+        assert_equal(302, response.status_code)
+        assert_equal('/filebrowser/view=s3a%3A%2F%2Fmy_bucket', response.url)
+      finally:
+        reset()

desktop/core/src/desktop/auth/backend.py

@@ -169,8 +169,7 @@ def get_groups(self):
     return self._get_profile().get_groups()
 
   def get_home_directory(self, force_home=False):
-    return REMOTE_STORAGE_HOME.get() if hasattr(REMOTE_STORAGE_HOME, 'get') and REMOTE_STORAGE_HOME.get() and not force_home \
-        else self._get_profile().home_directory
+    return self._get_profile().home_directory
 
   def has_hue_permission(self, action, app):
     return self._get_profile().has_hue_permission(action=action, app=app)

desktop/core/src/desktop/js/jquery/plugins/jquery.filechooser.js

@@ -82,7 +82,7 @@ const defaults = {
     s3a: {
       scheme: 's3a',
       root: 's3a://',
-      home: 's3a://',
+      home: '/?default_s3_home',
       icon: {
         brand: 'fa-cubes',
         home: 'fa-cubes'
@@ -810,6 +810,20 @@ Plugin.prototype.init = function () {
       '<div class="filechooser-container" style="position: relative"><div class="filechooser-services" style="position: absolute"></div><div class="filechooser-tree" style="width: 560px"></div></div>'
     );
   $.post('/filebrowser/api/get_filesystems', data => {
+    let initialHome = '/';
+    if (self.options.initialPath == '') {
+      if (
+        Object.keys(data.filesystems).length > 1 &&
+        Object.keys(data.filesystems).indexOf('hdfs') != -1
+      ) {
+        initialHome = '/?default_to_home';
+        self.options.fsSelected = 'hdfs';
+      } else {
+        const _scheme = Object.keys(data.filesystems)[0];
+        initialHome = defaults.filesysteminfo[_scheme]['home'];
+        self.options.fsSelected = _scheme;
+      }
+    }
     const initialPath = $.trim(self.options.initialPath);
     const scheme = initialPath && initialPath.substring(0, initialPath.indexOf(':'));
     if (data && data.status === 0) {
@@ -827,7 +841,7 @@ Plugin.prototype.init = function () {
         hueLocalStorage(STORAGE_PREFIX + self.options.user + self.options.fsSelected)
       );
     } else {
-      self.navigateTo('/?default_to_home');
+      self.navigateTo(initialHome);
     }
   });
 };

desktop/libs/aws/src/aws/s3/s3fs.py

@@ -35,6 +35,7 @@
 from aws.conf import get_default_region, get_locations, PERMISSION_ACTION_S3
 from aws.s3 import normpath, s3file, translate_s3_error, S3A_ROOT
 from aws.s3.s3stat import S3Stat
+from filebrowser.conf import REMOTE_STORAGE_HOME
 
 if sys.version_info[0] > 2:
   import urllib.request, urllib.error
@@ -82,6 +83,12 @@ def decorator(*args, **kwargs):
   return decorator
 
 
+def get_s3_home_directory():
+  return REMOTE_STORAGE_HOME.get() \
+         if hasattr(REMOTE_STORAGE_HOME, 'get') and REMOTE_STORAGE_HOME.get() \
+         else 's3a://'
+
+
 class S3FileSystem(object):
   def __init__(self, s3_connection, expiration=None, fs='s3a', headers=None, filebrowser_action=PERMISSION_ACTION_S3):
     self._s3_connection = s3_connection