[ozone] Add unit tests and update existing tests for ofs

apps/filebrowser/src/filebrowser/views_test.py

@@ -53,7 +53,7 @@
 from desktop.lib.django_test_util import make_logged_in_client
 from desktop.lib.test_utils import grant_access, add_to_group, add_permission, remove_from_group
 from desktop.lib.view_util import location_to_url
-from desktop.conf import is_oozie_enabled, RAZ
+from desktop.conf import is_oozie_enabled, RAZ, is_ofs_enabled, OZONE
 
 from hadoop import pseudo_hdfs4
 from hadoop.conf import UPLOAD_CHUNK_SIZE
@@ -1530,6 +1530,45 @@ def test_has_default_permissions(self):
     finally:
       remove_from_group(self.user.username, 'has_adls')
 
+class TestOFSAccessPermissions(object):
+  def setUp(self):
+    self.client = make_logged_in_client(username="test", groupname="default", recreate=True, is_superuser=False)
+    grant_access('test', 'test', 'filebrowser')
+    add_to_group('test')
+
+    self.user = User.objects.get(username="test")
+
+  def test_no_default_permissions(self):
+    response = self.client.get('/filebrowser/view=ofs://')
+    assert_equal(500, response.status_code)
+
+    response = self.client.get('/filebrowser/view=ofs://volume')
+    assert_equal(500, response.status_code)
+
+    response = self.client.get('/filebrowser/view=ofs://volume/bucket')
+    assert_equal(500, response.status_code)
+
+    response = self.client.get('/filebrowser/view=ofs://volume/bucket/hue')
+    assert_equal(500, response.status_code)
+
+    response = self.client.post('/filebrowser/rmtree', dict(path=['ofs://volume/bucket/hue']))
+    assert_equal(500, response.status_code)
+
+    # 500 for real currently
+    assert_raises(IOError, self.client.get, '/filebrowser/edit=ofs://volume/bucket/hue')
+
+  def test_has_default_permissions(self):
+    if not is_ofs_enabled():
+      raise SkipTest
+
+    add_permission(self.user.username, 'has_ofs', permname='ofs_access', appname='filebrowser')
+
+    try:
+      response = self.client.get('/filebrowser/view=ofs://')
+      assert_equal(200, response.status_code)
+    finally:
+      remove_from_group(self.user.username, 'has_ofs')
+
 
 class TestFileChooserRedirect(object):
 
@@ -1540,7 +1579,7 @@ def setUp(self):
 
     self.user = User.objects.get(username="test")
 
-  def test_hdfs_redirect(self):
+  def test_fs_redirect(self):
     with patch('desktop.lib.fs.proxyfs.ProxyFS.isdir') as is_dir:
       is_dir.return_value = True
 
@@ -1550,11 +1589,34 @@ def test_hdfs_redirect(self):
       assert_equal(302, response.status_code)
       assert_equal('/filebrowser/view=%2Fuser%2Ftest', response.url)
 
+      # OFS - default_ofs_home
+      reset = OZONE['default'].WEBHDFS_URL.set_for_testing(None)
+      try:
+        response = self.client.get('/filebrowser/view=%2F?default_ofs_home')
+
+        assert_equal(302, response.status_code)
+        assert_equal('/filebrowser/view=ofs%3A%2F%2F', response.url)
+      finally:
+        reset()
+
+      reset = OZONE['default'].WEBHDFS_URL.set_for_testing('http://localhost:9778/webhdfs/v1')
+      try:
+        response = self.client.get('/filebrowser/view=%2F?default_ofs_home')
+
+        assert_equal(302, response.status_code)
+        assert_equal('/filebrowser/view=ofs%3A%2F%2F', response.url)
+      finally:
+        reset()
+
       # ABFS - default_abfs_home
-      response = self.client.get('/filebrowser/view=%2F?default_abfs_home')
+      reset = ABFS_CLUSTERS['default'].FS_DEFAULTFS.set_for_testing(None)
+      try:
+        response = self.client.get('/filebrowser/view=%2F?default_abfs_home')
 
-      assert_equal(302, response.status_code)
-      assert_equal('/filebrowser/view=abfs%3A%2F%2F', response.url)
+        assert_equal(302, response.status_code)
+        assert_equal('/filebrowser/view=abfs%3A%2F%2F', response.url)
+      finally:
+        reset()
 
       reset = ABFS_CLUSTERS['default'].FS_DEFAULTFS.set_for_testing('abfs://data-container@mystorage.dfs.core.windows.net')
       try:
@@ -1592,10 +1654,14 @@ def test_hdfs_redirect(self):
           reset()
 
       # S3A - default_s3_home
-      response = self.client.get('/filebrowser/view=%2F?default_s3_home')
+      reset = REMOTE_STORAGE_HOME.set_for_testing(None)
+      try:
+        response = self.client.get('/filebrowser/view=%2F?default_s3_home')
 
-      assert_equal(302, response.status_code)
-      assert_equal('/filebrowser/view=s3a%3A%2F%2F', response.url)
+        assert_equal(302, response.status_code)
+        assert_equal('/filebrowser/view=s3a%3A%2F%2F', response.url)
+      finally:
+        reset()
 
       reset = REMOTE_STORAGE_HOME.set_for_testing('s3a://my_bucket')
       try:
@@ -1683,3 +1749,19 @@ def test_s3a_correction_already_correct(self):
 
     normalized = _normalize_path(path)
     assert_equal(path, normalized)
+
+  def test_ofs_correction(self):
+    path = 'ofs:%2Fsome%2Fpath'
+    expected_corrected_path = 'ofs://some/path'
+
+    normalized_once = _normalize_path(path)
+    assert_equal(expected_corrected_path, normalized_once)
+
+    normalized_twice = _normalize_path(normalized_once)
+    assert_equal(expected_corrected_path, normalized_twice)
+
+  def test_ofs_correction_already_correct(self):
+    path = 'ofs://some/path'
+
+    normalized = _normalize_path(path)
+    assert_equal(path, normalized)

desktop/core/src/desktop/lib/fs/ozone/ofs_test.py

@@ -0,0 +1,116 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from nose.tools import assert_equal
+
+from desktop import conf
+from desktop.lib.fs.ozone.client import _make_ofs_client
+
+
+class TestOFSClient(object):
+
+  @classmethod
+  def setUpClass(cls):
+    cls._resets = [
+      conf.OZONE['default'].FS_DEFAULTFS.set_for_testing('ofs://ozone1'),
+      conf.OZONE['default'].LOGICAL_NAME.set_for_testing('test-logical-name'),
+      conf.OZONE['default'].WEBHDFS_URL.set_for_testing('http://gethue-ozone:9778/webhdfs/v1'),
+      conf.OZONE['default'].SECURITY_ENABLED.set_for_testing(True),
+      conf.OZONE['default'].SSL_CERT_CA_VERIFY.set_for_testing(True),
+      conf.OZONE['default'].TEMP_DIR.set_for_testing('/tmp')
+    ]
+
+    cls.ofs_client = _make_ofs_client('default')
+
+
+  def test_client_attributes(self):
+    assert_equal(self.ofs_client._url, 'http://gethue-ozone:9778/webhdfs/v1')
+    assert_equal(self.ofs_client._superuser, None)
+    assert_equal(self.ofs_client._security_enabled, True)
+    assert_equal(self.ofs_client._ssl_cert_ca_verify, True)
+    assert_equal(self.ofs_client._temp_dir, '/tmp')
+    assert_equal(self.ofs_client._umask, 530)
+    assert_equal(self.ofs_client._fs_defaultfs, 'ofs://ozone1')
+    assert_equal(self.ofs_client._logical_name, 'test-logical-name')
+    assert_equal(self.ofs_client._supergroup, None)
+    assert_equal(self.ofs_client._scheme, 'ofs')
+    assert_equal(self.ofs_client._netloc, 'ozone1')
+    assert_equal(self.ofs_client._is_remote, True)
+    assert_equal(self.ofs_client._has_trash_support, False)
+    assert_equal(self.ofs_client.expiration, None)
+    assert_equal(self.ofs_client._filebrowser_action, 'ofs_access')
+
+
+  def test_strip_normpath(self):
+    test_path = self.ofs_client.strip_normpath('ofs://vol1/buk1/key')
+    assert_equal(test_path, '/vol1/buk1/key')
+
+    test_path = self.ofs_client.strip_normpath('ofs:/vol1/buk1/key')
+    assert_equal(test_path, '/vol1/buk1/key')
+
+    test_path = self.ofs_client.strip_normpath('/vol1/buk1/key')
+    assert_equal(test_path, '/vol1/buk1/key')
+
+
+  def test_normpath(self):
+    test_path = self.ofs_client.normpath('ofs://')
+    assert_equal(test_path, 'ofs://')
+
+    test_path = self.ofs_client.normpath('ofs://vol1/buk1/key')
+    assert_equal(test_path, 'ofs://vol1/buk1/key')
+
+    test_path = self.ofs_client.normpath('ofs://vol1/buk1/key/')
+    assert_equal(test_path, 'ofs://vol1/buk1/key')
+
+    test_path = self.ofs_client.normpath('ofs://vol1/buk1/key//')
+    assert_equal(test_path, 'ofs://vol1/buk1/key')
+
+    test_path = self.ofs_client.normpath('ofs://vol1/buk1//key//')
+    assert_equal(test_path, 'ofs://vol1/buk1/key')
+
+
+  def test_isroot(self):
+    is_root = self.ofs_client.isroot('ofs://vol1/buk1/key')
+    assert_equal(is_root, False)
+
+    is_root = self.ofs_client.isroot('ofs://')
+    assert_equal(is_root, True)
+
+
+  def test_parent_path(self):
+    parent_path = self.ofs_client.parent_path('ofs://')
+    assert_equal(parent_path, 'ofs://')
+
+    parent_path = self.ofs_client.parent_path('ofs://vol1/buk1/dir1/file1.csv')
+    assert_equal(parent_path, 'ofs://vol1/buk1/dir1')
+
+    parent_path = self.ofs_client.parent_path('ofs://vol1/buk1/key')
+    assert_equal(parent_path, 'ofs://vol1/buk1')
+
+    parent_path = self.ofs_client.parent_path('ofs://vol1/buk1')
+    assert_equal(parent_path, 'ofs://vol1/')
+
+    parent_path = self.ofs_client.parent_path('ofs://vol1')
+    assert_equal(parent_path, 'ofs://')
+
+
+  @classmethod
+  def tearDownClass(cls):
+    for reset in cls._resets:
+      reset()
+
+

desktop/core/src/desktop/lib/fs/ozone/ofsstat_test.py

@@ -0,0 +1,56 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from nose.tools import assert_equal
+from desktop.lib.fs.ozone.ofsstat import OzoneFSStat
+
+
+class TestOzoneFSStat(object):
+  def setUp(self):
+    test_file_status = {
+      'pathSuffix': 'testfile.csv', 'type': 'FILE', 'length': 32, 'owner': 'hueadmin', 'group': 'huegroup',
+      'permission': '666', 'accessTime': 1677914460588, 'modificationTime': 1677914460588, 'blockSize': 268435456, 'replication': 3}
+
+    test_parent_path = '/gethue/'
+
+    self.stat = OzoneFSStat(test_file_status, test_parent_path)
+
+
+  def test_stat_attributes(self):
+    assert_equal(self.stat.name, 'testfile.csv')
+    assert_equal(self.stat.path, 'ofs://gethue/testfile.csv')
+    assert_equal(self.stat.isDir, False)
+    assert_equal(self.stat.type, 'FILE')
+    assert_equal(self.stat.atime, 1677914460)
+    assert_equal(self.stat.mtime, 1677914460)
+    assert_equal(self.stat.user, 'hueadmin')
+    assert_equal(self.stat.group, 'huegroup')
+    assert_equal(self.stat.size, 32)
+    assert_equal(self.stat.blockSize, 268435456)
+    assert_equal(self.stat.replication, 3)
+    assert_equal(self.stat.aclBit, None)
+    assert_equal(self.stat.fileId, None)
+    assert_equal(self.stat.mode, 33206)
+
+
+  def test_to_json_dict(self):
+    expected_json_dict = {
+      'path': 'ofs://gethue/testfile.csv', 'size': 32, 'atime': 1677914460, 'mtime': 1677914460, 'mode': 33206, 'user': 'hueadmin',
+      'group': 'huegroup', 'blockSize': 268435456, 'replication': 3}
+
+    assert_equal(self.stat.to_json_dict(), expected_json_dict)