[hplsql] removing v1 code and improving v2 code (#2672)

apps/beeswax/src/beeswax/conf.py

@@ -125,12 +125,6 @@ def get_hive_thrift_http_port():
   dynamic_default=get_hive_thrift_http_port,
   type=int)
 
-HPLSQL = Config(
-  key="hplsql",
-  default=False,
-  type=coerce_bool,
-  help=_t('Enable the HPLSQL mode.'))
-
 HIVE_METASTORE_HOST = Config(
   key="hive_metastore_host",
   help=_t("Host where Hive Metastore Server (HMS) is running. If Kerberos security is enabled, "

apps/beeswax/src/beeswax/design.py

@@ -83,10 +83,10 @@ class HQLdesign(object):
   want to use "$" natively, but we leave that as an advanced
   option to turn off.
   """
-  _QUERY_ATTRS = [ 'query', 'type', 'is_parameterized', 'email_notify', 'database' ]
-  _SETTINGS_ATTRS = [ 'key', 'value' ]
-  _FILE_RES_ATTRS = [ 'type', 'path' ]
-  _FUNCTIONS_ATTRS = [ 'name', 'class_name' ]
+  _QUERY_ATTRS = ['query', 'type', 'is_parameterized', 'email_notify', 'database']
+  _SETTINGS_ATTRS = ['key', 'value']
+  _FILE_RES_ATTRS = ['type', 'path']
+  _FUNCTIONS_ATTRS = ['name', 'class_name']
 
   def __init__(self, form=None, query_type=None):
     """Initialize the design from a valid form data."""
@@ -136,7 +136,10 @@ def statement_count(self):
   @property
   def statements(self):
     hql_query = strip_trailing_semicolon(self.hql_query)
-    return [strip_trailing_semicolon(statement) for (start_row, start_col), (end_row, end_col), statement in split_statements(hql_query)]
+    dialect = 'hplsql' if self._data_dict['query']['type'] == 4 else None
+    return [
+      strip_trailing_semicolon(statement) for (start_row, start_col), (end_row, end_col), statement in split_statements(hql_query, dialect)
+    ]
 
   @staticmethod
   def get_default_data_dict():
@@ -248,7 +251,7 @@ def normalize_form_dict(form, attr_list):
   Each attr is a field name. And the value is obtained by looking up the form's data dict.
   """
   assert isinstance(form, forms.Form)
-  res = { }
+  res = {}
   for attr in attr_list:
     res[attr] = form.cleaned_data.get(attr)
   return res
@@ -259,7 +262,7 @@ def normalize_formset_dict(formset, attr_list):
   normalize_formset_dict(formset, attr_list) -> A list of dictionary of (attr, value)
   """
   assert isinstance(formset, BaseSimpleFormSet)
-  res = [ ]
+  res = []
   for form in formset.forms:
     res.append(normalize_form_dict(form, attr_list))
   return res

apps/beeswax/src/beeswax/models.py

@@ -31,7 +31,6 @@
 from enum import Enum
 from TCLIService.ttypes import TSessionHandle, THandleIdentifier, TOperationState, TOperationHandle, TOperationType
 
-from beeswax.conf import HPLSQL
 from desktop.lib.exceptions_renderable import PopupException
 from desktop.models import Document, Document2
 from desktop.redaction import global_redaction_engine
@@ -53,7 +52,7 @@
 # Constants for DB fields, hue ini
 BEESWAX = 'beeswax'
 HIVE_SERVER2 = 'hiveserver2'
-QUERY_TYPES = (HQL, IMPALA, RDBMS, SPARK) = list(range(4))
+QUERY_TYPES = (HQL, IMPALA, RDBMS, SPARK, HPLSQL) = list(range(5))
 
 class QueryHistory(models.Model):
   """
@@ -271,7 +270,7 @@ class SavedQuery(models.Model):
   DEFAULT_NEW_DESIGN_NAME = _('My saved query')
   AUTO_DESIGN_SUFFIX = _(' (new)')
   TYPES = QUERY_TYPES
-  TYPES_MAPPING = {'beeswax': HQL, 'hql': HQL, 'impala': IMPALA, 'rdbms': RDBMS, 'spark': SPARK}
+  TYPES_MAPPING = {'beeswax': HQL, 'hql': HQL, 'impala': IMPALA, 'rdbms': RDBMS, 'spark': SPARK, 'hplsql': HPLSQL}
 
   type = models.IntegerField(null=False)
   owner = models.ForeignKey(User, on_delete=models.CASCADE, db_index=True)
@@ -400,8 +399,6 @@ def save(self, *args, **kwargs):
 class SessionManager(models.Manager):
 
   def get_session(self, user, application='beeswax', filter_open=True):
-    if HPLSQL.get() and application == 'beeswax':
-      application = 'hplsql'
     try:
       q = self.filter(owner=user, application=application).exclude(guid='').exclude(secret='')
       if filter_open:

apps/beeswax/src/beeswax/server/hive_server2_lib.py

@@ -34,7 +34,7 @@
 
 from beeswax import conf as beeswax_conf, hive_site
 from beeswax.hive_site import hiveserver2_use_ssl
-from beeswax.conf import CONFIG_WHITELIST, LIST_PARTITIONS_LIMIT, HPLSQL, MAX_CATALOG_SQL_ENTRIES
+from beeswax.conf import CONFIG_WHITELIST, LIST_PARTITIONS_LIMIT, MAX_CATALOG_SQL_ENTRIES
 from beeswax.models import Session, HiveServerQueryHandle, HiveServerQueryHistory
 from beeswax.server.dbms import Table, DataTable, QueryServerException, InvalidSessionQueryServerException
 
@@ -674,8 +674,6 @@ def open_session(self, user):
 
     if self.query_server['server_name'] == 'beeswax': # All the time
       kwargs['configuration'].update({'hive.server2.proxy.user': user.username})
-      if HPLSQL.get():
-        kwargs['configuration'].update({'set:hivevar:mode': 'HPLSQL'})
 
     if self.query_server['server_name'] == 'hplsql': # All the time
       kwargs['configuration'].update({'hive.server2.proxy.user': user.username, 'set:hivevar:mode': 'HPLSQL'})
@@ -710,11 +708,10 @@ def open_session(self, user):
 
     encoded_status, encoded_guid = HiveServerQueryHandle(secret=sessionId.secret, guid=sessionId.guid).get()
     properties = json.dumps(res.configuration)
-    application = 'hplsql' if HPLSQL.get() and self.query_server['server_name'] == 'beeswax' else self.query_server['server_name']
 
     session = Session.objects.create(
         owner=user,
-        application=application,
+        application=self.query_server['server_name'],
         status_code=res.status.statusCode,
         secret=encoded_status,
         guid=encoded_guid,
@@ -1000,10 +997,7 @@ def execute_async_query(self, query, statement=0, session=None):
 
     # The query can override the default configuration
     configuration.update(self._get_query_configuration(query))
-    if self.query_server['server_name'] == 'hplsql':
-      query_statement = query.hql_query
-    else:
-      query_statement = query.get_query_statement(statement)
+    query_statement = query.get_query_statement(statement)
 
     return self.execute_async_statement(statement=query_statement, conf_overlay=configuration, session=session)
 

desktop/conf.dist/hue.ini

@@ -1390,9 +1390,6 @@ submit_to=True
 # Use SASL framework to establish connection to host.
 ## use_sasl=false
 
-# Enable the HPLSQL mode.
-## hplsql=false
-
 # Max number of objects (columns, tables, databases) available to list in the left assist, autocomplete, table browser etc.
 # Setting this higher than the default can degrade performance.
 ## max_catalog_sql_entries=5000

desktop/conf/pseudo-distributed.ini.tmpl

@@ -1373,9 +1373,6 @@
   # Use SASL framework to establish connection to host.
   ## use_sasl=false
 
-  # Enable the HPLSQL mode.
-  ## hplsql=false
-
   # Max number of objects (columns, tables, databases) available to list in the left assist, autocomplete, table browser etc.
   # Setting this higher than the default can degrade performance.
   ## max_catalog_sql_entries=5000

desktop/core/src/desktop/js/apps/editor/EditorViewModel.js

@@ -88,13 +88,9 @@ export default class EditorViewModel {
     });
 
     this.editorType = ko.pureComputed(() => this.activeConnector() && this.activeConnector().id);
-    this.editorTitle = ko.pureComputed(() => {
-      return window.HPLSQL &&
-        this.activeConnector() &&
-        this.activeConnector().displayName === 'Hive'
-        ? 'Hive HPL/SQL'
-        : this.activeConnector() && this.activeConnector().displayName;
-    });
+    this.editorTitle = ko.pureComputed(
+      () => this.activeConnector() && this.activeConnector().displayName
+    );
 
     this.editorIcon = ko.pureComputed(() => {
       const dialect = this.activeConnector()?.dialect;

desktop/core/src/desktop/js/apps/notebook/NotebookViewModel.js

@@ -168,10 +168,7 @@ export default class NotebookViewModel {
       const foundInterpreter = options.languages.find(
         interpreter => interpreter.type === self.editorType()
       );
-      return window.HPLSQL &&
-        (foundInterpreter?.displayName || foundInterpreter?.name || self.editorType()) === 'Hive'
-        ? 'Hive HPL/SQL'
-        : foundInterpreter?.displayName || foundInterpreter?.name || self.editorType();
+      return foundInterpreter?.displayName || foundInterpreter?.name || self.editorType();
     });
     self.autocompleteTimeout = options.autocompleteTimeout;
     self.selectedNotebook = ko.observable();

desktop/core/src/desktop/templates/global_js_constants.mako

@@ -24,7 +24,7 @@
       VCS, ENABLE_GIST, ENABLE_LINK_SHARING, has_channels, has_connectors, ENABLE_UNIFIED_ANALYTICS, RAZ
   from desktop.models import hue_version, _get_apps, get_cluster_config, _handle_user_dir_raz
 
-  from beeswax.conf import DOWNLOAD_BYTES_LIMIT, DOWNLOAD_ROW_LIMIT, LIST_PARTITIONS_LIMIT, CLOSE_SESSIONS, HPLSQL
+  from beeswax.conf import DOWNLOAD_BYTES_LIMIT, DOWNLOAD_ROW_LIMIT, LIST_PARTITIONS_LIMIT, CLOSE_SESSIONS
   from dashboard.conf import HAS_SQL_ENABLED
   from jobbrowser.conf import ENABLE_HISTORY_V2
   from filebrowser.conf import SHOW_UPLOAD_BUTTON, REMOTE_STORAGE_HOME
@@ -141,7 +141,6 @@
   window.JB_SINGLE_CHECK_INTERVAL_IN_MILLIS = 5000;
   window.JB_MULTI_CHECK_INTERVAL_IN_MILLIS = 20000;
   window.CLOSE_SESSIONS = {'hive': '${ CLOSE_SESSIONS.get() }' === 'True'};
-  window.HPLSQL = '${ HPLSQL.get() }' === 'True';
 
   window.HUE_URLS = {
     IMPORTER_CREATE_TABLE: '${ 'indexer' in apps and url('indexer:importer_prefill', source_type = 'all', target_type = 'table')}',

desktop/libs/notebook/src/notebook/connectors/hiveserver2.py

@@ -57,7 +57,7 @@
   from beeswax import conf as beeswax_conf, data_export
   from beeswax.api import _autocomplete, _get_sample_data
   from beeswax.conf import CONFIG_WHITELIST as hive_settings, DOWNLOAD_ROW_LIMIT, DOWNLOAD_BYTES_LIMIT, MAX_NUMBER_OF_SESSIONS, \
-      has_session_pool, has_multiple_sessions, CLOSE_SESSIONS, HPLSQL
+      has_session_pool, has_multiple_sessions, CLOSE_SESSIONS
   from beeswax.data_export import upload
   from beeswax.design import hql_query
   from beeswax.models import QUERY_TYPES, HiveServerQueryHandle, HiveServerQueryHistory, QueryHistory, Session
@@ -180,8 +180,6 @@ def get_properties(lang='hive'):
   @query_error_handler
   def create_session(self, lang='hive', properties=None):
     application = 'beeswax' if lang == 'hive' or lang == 'llap' else lang
-    if application == 'beeswax' and HPLSQL.get():
-      application = 'hplsql'
 
     if has_session_pool():
       session = Session.objects.get_tez_session(self.user, application, MAX_NUMBER_OF_SESSIONS.get())
@@ -694,8 +692,6 @@ def _get_session_by_id(self, notebook, type='hive'):
       session_id = session.get('id')
       if session_id:
         filters = {'id': session_id, 'application': 'beeswax' if type == 'hive' or type == 'llap' else type}
-        if HPLSQL.get() and filters['application'] == 'beeswax':
-          filters['application'] = 'hplsql'
         if not is_admin(self.user):
           filters['owner'] = self.user
         return Session.objects.get(**filters)
@@ -741,10 +737,11 @@ def _prepare_hql_query(self, snippet, statement, session):
       functions = next((prop['value'] for prop in properties if prop['key'] == 'functions'), None)
 
     database = snippet.get('database') or 'default'
+    query_type = QUERY_TYPES[4] if hasattr(snippet, 'dialect') and snippet['dialect'] == 'hplsql' else QUERY_TYPES[0]
 
     return hql_query(
       statement,
-      query_type=QUERY_TYPES[0],
+      query_type=query_type,
       settings=settings,
       file_resources=file_resources,
       functions=functions,

desktop/libs/notebook/src/notebook/sql_utils.py

@@ -22,7 +22,6 @@
 import re
 import sys
 
-from beeswax.conf import HPLSQL
 from desktop.lib.i18n import smart_str
 
 if sys.version_info[0] > 2:
@@ -32,12 +31,12 @@
 
 
 # Note: Might be replaceable by sqlparse.split
-def get_statements(hql_query):
+def get_statements(hql_query, dialect=None):
   hql_query = strip_trailing_semicolon(hql_query)
   hql_query_sio = string_io(hql_query)
 
   statements = []
-  for (start_row, start_col), (end_row, end_col), statement in split_statements(hql_query_sio.read()):
+  for (start_row, start_col), (end_row, end_col), statement in split_statements(hql_query_sio.read(), dialect):
     statements.append({
       'start': {
         'row': start_row,
@@ -51,33 +50,14 @@ def get_statements(hql_query):
     })
   return statements
 
-def get_hplsql_statements(hplsql_query):
-  statements = []
-  statements.append(
-    {
-      'start': {
-        'row': 0,
-        'column': 0
-      },
-      'end': {
-        'row': 0,
-        'column': len(hplsql_query) - 1
-      },
-      'statement': strip_trailing_semicolon(hplsql_query.rstrip())
-    })
-  return statements
-
 def get_current_statement(snippet):
   # Multiquery, if not first statement or arrived to the last query
   should_close = False
   handle = snippet['result'].get('handle', {})
   statement_id = handle.get('statement_id', 0)
   statements_count = handle.get('statements_count', 1)
 
-  if HPLSQL.get() and snippet['dialect'] == 'hive':
-    statements = get_hplsql_statements(snippet['statement'])
-  else:
-    statements = get_statements(snippet['statement'])
+  statements = get_statements(snippet['statement'], snippet['dialect'] if hasattr(snippet, 'dialect') else None)
 
   statement_id = min(statement_id, len(statements) - 1) # In case of removal of statements
   previous_statement_hash = compute_statement_hash(statements[statement_id]['statement'])
@@ -111,7 +91,7 @@ def compute_statement_hash(statement):
   else:
     return hashlib.sha224(smart_str(statement)).hexdigest()
 
-def split_statements(hql):
+def split_statements(hql, dialect=None):
   """
   Split statements at semicolons ignoring the ones inside quotes and comments.
   The comment symbols that come inside quotes should be ignored.
@@ -126,7 +106,7 @@ def split_statements(hql):
   end_row = 0
   end_col = len(hql) - 1
 
-  if hql.find(';') in (-1, len(hql) - 1):
+  if hql.find(';') in (-1, len(hql) - 1) or dialect == 'hplsql':
     return [((start_row, start_col), (end_row, end_col), hql)]
 
   lines = hql.splitlines()

desktop/libs/notebook/src/notebook/sql_utils_tests.py

@@ -17,9 +17,9 @@
 # limitations under the License.
 
 from beeswax.design import hql_query
-from notebook.sql_utils import strip_trailing_semicolon, split_statements, get_hplsql_statements
+from notebook.sql_utils import strip_trailing_semicolon, split_statements
 
-from nose.tools import assert_equal
+from nose.tools import assert_equal, assert_not_equal
 
 
 def test_split_statements():
@@ -53,5 +53,10 @@ def test_get_hplsql_statements():
   # Not spliting statements at semicolon
   assert_equal(
     "CREATE FUNCTION hello()\n RETURNS STRING\nBEGIN\n RETURN 'Hello, world';\nEND",
-    get_hplsql_statements("CREATE FUNCTION hello()\n RETURNS STRING\nBEGIN\n RETURN 'Hello, world';\nEND;")[0]['statement']
+    split_statements("CREATE FUNCTION hello()\n RETURNS STRING\nBEGIN\n RETURN 'Hello, world';\nEND", 'hplsql')[0][2]
+  )
+
+  assert_not_equal(
+    "CREATE FUNCTION hello()\n RETURNS STRING\nBEGIN\n RETURN 'Hello, world';\nEND",
+    split_statements("CREATE FUNCTION hello()\n RETURNS STRING\nBEGIN\n RETURN 'Hello, world';\nEND")[0][2]
   )