HUE-8737 [core] Fix beeswax, hdfs, oozie and desktop libs py3 compati…

…ble code

apps/beeswax/src/beeswax/hive_site.py

@@ -25,13 +25,18 @@
 import os.path
 import re
 import socket
+import sys
 
 from desktop.lib import security_util
 from hadoop import confparse
 from hadoop.ssl_client_site import get_trustore_location, get_trustore_password
 
 import beeswax.conf
 
+if sys.version_info[0] > 2:
+  open_file = open
+else:
+  open_file = file
 
 LOG = logging.getLogger(__name__)
 
@@ -196,7 +201,7 @@ def _parse_hive_site():
 
   _HIVE_SITE_PATH = os.path.join(beeswax.conf.HIVE_CONF_DIR.get(), 'hive-site.xml')
   try:
-    data = file(_HIVE_SITE_PATH, 'r').read()
+    data = open_file(_HIVE_SITE_PATH, 'r').read()
   except IOError as err:
     if err.errno != errno.ENOENT:
       LOG.error('Cannot read from "%s": %s' % (_HIVE_SITE_PATH, err))
@@ -211,4 +216,4 @@ def get_hive_site_content():
   if not os.path.exists(hive_site_path):
     return ''
   else:
-    return file(hive_site_path, 'r').read()
+    return open_file(hive_site_path, 'r').read()

apps/beeswax/src/beeswax/server/dbms.py

@@ -15,17 +15,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from builtins import str
 from builtins import object
 import logging
 import re
+import sys
 import threading
 import time
 import json
 
 from django.core.cache import caches
 from django.urls import reverse
-from django.utils.encoding import force_unicode
 from django.utils.translation import ugettext as _
 from kazoo.client import KazooClient
 
@@ -50,6 +49,12 @@
 from beeswax.models import QueryHistory, QUERY_TYPES
 
 
+if sys.version_info[0] > 2:
+    from django.utils.encoding import force_text as force_unicode
+else:
+    from django.utils.encoding import force_unicode
+
+
 LOG = logging.getLogger(__name__)
 
 DBMS_CACHE = {}

apps/oozie/src/oozie/models.py

@@ -39,7 +39,7 @@
 from django.contrib.contenttypes.fields import GenericRelation
 from django.contrib.contenttypes.models import ContentType
 from django.forms.models import inlineformset_factory
-from django.utils.encoding import force_unicode, smart_str
+from django.utils.encoding import smart_str
 from django.utils.translation import ugettext as _, ugettext_lazy as _t
 import django.utils.timezone as dtz
 
@@ -61,8 +61,10 @@
 
 if sys.version_info[0] > 2:
   from io import StringIO as string_io
+  from django.utils.encoding import force_text as force_unicode
 else:
   from cStringIO import StringIO as string_io
+  from django.utils.encoding import force_unicode
 
 
 LOG = logging.getLogger(__name__)

apps/oozie/src/oozie/models2.py

@@ -24,6 +24,7 @@
 import logging
 import os
 import re
+import sys
 import time
 import uuid
 
@@ -34,7 +35,6 @@
 
 from django.urls import reverse
 from django.db.models import Q
-from django.utils.encoding import force_unicode
 from django.utils.translation import ugettext as _
 
 from desktop.conf import USE_DEFAULT_CONFIGURATION
@@ -57,6 +57,11 @@
 from oozie.utils import utc_datetime_format, UTC_TIME_FORMAT, convert_to_server_timezone
 from oozie.importlib.workflows import generate_v2_graph_nodes, MalformedWfDefException, InvalidTagWithNamespaceException
 
+if sys.version_info[0] > 2:
+    from django.utils.encoding import force_text as force_unicode
+else:
+    from django.utils.encoding import force_unicode
+
 WORKFLOW_DEPTH_LIMIT = 24
 LOG = logging.getLogger(__name__)
 

desktop/core/src/desktop/lib/django_forms.py

@@ -33,16 +33,18 @@
 from django.forms.widgets import MultiWidget, Select, TextInput, Textarea, HiddenInput, Input
 from django.utils import formats
 from django.utils.safestring import mark_safe
-from django.utils.encoding import python_2_unicode_compatible, force_unicode
+from django.utils.encoding import python_2_unicode_compatible
 
 import desktop.lib.i18n
 from desktop.lib.i18n import smart_str
 
 if sys.version_info[0] > 2:
   import urllib.request, urllib.error
   from urllib.parse import quote_plus as urllib_quote_plus
+  from django.utils.encoding import force_text as force_unicode
 else:
   from urllib import quote_plus as urllib_quote_plus
+  from django.utils.encoding import force_unicode
 
 LOG = logging.getLogger(__name__)
 

desktop/core/src/desktop/lib/exceptions_renderable.py

@@ -23,7 +23,10 @@
 import sys
 import traceback
 
-from django.utils.encoding import force_unicode
+if sys.version_info[0] > 2:
+    from django.utils.encoding import force_text as force_unicode
+else:
+    from django.utils.encoding import force_unicode
 
 import desktop.lib.django_util
 

desktop/core/src/desktop/lib/i18n.py

@@ -59,7 +59,7 @@ def smart_unicode(s, strings_only=False, errors='strict', encoding=None):
   Wrapper around Django's version, while supplying our configured encoding.
   Decode char array to unicode.
   """
-  return django.utils.encoding.smart_unicode(
+  return django.utils.encoding.smart_text(
         s, encoding if encoding is not None else get_site_encoding(), strings_only, errors)
 
 def force_unicode(s, strings_only=False, errors='strict'):

desktop/libs/aws/src/aws/s3/__init__.py

@@ -16,6 +16,7 @@
 from __future__ import absolute_import
 
 from builtins import map
+from future.utils import raise_
 import calendar
 import errno
 import logging
@@ -55,7 +56,7 @@ def wrapped(*args, **kwargs):
       _, exc, tb = sys.exc_info()
       logging.error('S3 error: %s' % exc)
       lookup = lookup_s3error(exc)
-      raise lookup.__class__, lookup, tb
+      raise_(lookup.__class__, lookup, tb)
   return wrapped
 
 

desktop/libs/aws/src/aws/s3/s3fs.py

@@ -23,7 +23,7 @@
 import os
 import posixpath
 import re
-from urlparse import urlparse
+import sys
 import time
 
 from boto.exception import BotoClientError, S3ResponseError
@@ -38,6 +38,12 @@
 from aws.s3 import normpath, s3file, translate_s3_error, S3A_ROOT
 from aws.s3.s3stat import S3Stat
 
+if sys.version_info[0] > 2:
+  import urllib.request, urllib.error
+  from urllib.parse import quote as urllib_quote, urlparse as lib_urlparse
+else:
+  from urllib import quote as urllib_quote
+  from urlparse import urlparse as lib_urlparse
 
 DEFAULT_READ_SIZE = 1024 * 1024  # 1MB
 BUCKET_NAME_PATTERN = re.compile("^((?:(?:[a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9_\-]*[a-zA-Z0-9])\.)*(?:[A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9_\-]*[A-Za-z0-9]))$")
@@ -205,7 +211,7 @@ def _cut_separator(path):
 
   @staticmethod
   def isroot(path):
-    parsed = urlparse(path)
+    parsed = lib_urlparse(path)
     return (parsed.path == '/' or parsed.path == '') and parsed.netloc == ''
 
   @staticmethod

desktop/libs/azure/src/azure/abfs/abfs.py

@@ -23,13 +23,12 @@
 from builtins import object
 import logging
 import os
+import sys
 import threading
 import re
 
 from math import ceil
 from posixpath import join
-from urllib.parse import urlparse
-from urllib import quote
 
 from hadoop.hdfs_site import get_umask_mode
 from hadoop.fs.exceptions import WebHdfsException
@@ -41,6 +40,13 @@
 from azure.abfs.abfsstats import ABFSStat
 from azure.conf import PERMISSION_ACTION_ABFS
 
+if sys.version_info[0] > 2:
+  import urllib.request, urllib.error
+  from urllib.parse import quote as urllib_quote
+  from urllib.parse import urlparse as lib_urlparse
+else:
+  from urlparse import urlparse as lib_urlparse
+  from urllib import quote as urllib_quote
 
 LOG = logging.getLogger(__name__)
 
@@ -75,7 +81,7 @@ def __init__(self, url,
     self._logical_name = logical_name
     self._supergroup = hdfs_supergroup
     self._auth_provider = auth_provider
-    split = urlparse(fs_defaultfs)
+    split = lib_urlparse(fs_defaultfs)
     self._scheme = split.scheme
     self._netloc = split.netloc
     self._is_remote = True
@@ -509,7 +515,7 @@ def rename(self, old, new):
     Renames a file
     """ 
     LOG.debug("%s\n%s" % (old, new))
-    headers = {'x-ms-rename-source' : '/' + quote(Init_ABFS.strip_scheme(old)) }
+    headers = {'x-ms-rename-source' : '/' + urllib_quote(Init_ABFS.strip_scheme(old)) }
     try:
       self._create_path(new, headers=headers, overwrite=True)
     except WebHdfsException as e:

desktop/libs/dashboard/src/dashboard/api.py

@@ -19,9 +19,9 @@
 import hashlib
 import json
 import logging
+import sys
 import uuid
 
-from django.utils.encoding import force_unicode
 from django.utils.translation import ugettext as _
 
 from desktop.conf import ENABLE_DOWNLOAD
@@ -46,6 +46,10 @@
 from dashboard.models import Collection2, augment_solr_response, pairwise2, augment_solr_exception,\
   NESTED_FACET_FORM, COMPARE_FACET, QUERY_FACET, extract_solr_exception_message
 
+if sys.version_info[0] > 2:
+    from django.utils.encoding import force_text as force_unicode
+else:
+    from django.utils.encoding import force_unicode
 
 LOG = logging.getLogger(__name__)
 

desktop/libs/hadoop/src/hadoop/fs/hadoopfs.py

@@ -36,7 +36,7 @@
 import subprocess
 import sys
 
-from django.utils.encoding import smart_str, force_unicode
+from django.utils.encoding import smart_str
 from django.utils.translation import ugettext as _
 
 from desktop.lib import i18n
@@ -46,8 +46,10 @@
 from hadoop.fs.exceptions import PermissionDeniedException
 
 if sys.version_info[0] > 2:
+  from django.utils.encoding import force_text as force_unicode
   from urllib.parse import urlsplit as lib_urlsplit
 else:
+  from django.utils.encoding import force_unicode
   from urlparse import urlsplit as lib_urlsplit
 
 LOG = logging.getLogger(__name__)

desktop/libs/hadoop/src/hadoop/fs/webhdfs.py

@@ -78,7 +78,7 @@ def __init__(
       security_enabled=False,
       ssl_cert_ca_verify=True,
       temp_dir="/tmp",
-      umask=01022,
+      umask=0o1022,
       hdfs_supergroup=None):
     self._url = url
     self._superuser = hdfs_superuser