Merge branch 'hhyo:master' into master

hhyo · Feb 10, 2022 · 61c9d32 · 61c9d32
2 parents 60219bd + 0abcd43
commit 61c9d32
Show file tree

Hide file tree

Showing 29 changed files with 187 additions and 104 deletions.
diff --git a/common/static/dist/js/utils.js b/common/static/dist/js/utils.js
@@ -0,0 +1,7 @@
+var onLoadErrorCallback = function (status, jqXHR) {
+    if (status === 403) {
+        alert("权限错误，您没有权限查看该数据！");
+    } else {
+        alert("未知错误，请联系管理员处理！");
+    }
+};
diff --git a/common/templates/base.html b/common/templates/base.html
@@ -301,6 +301,7 @@
 <script src="{% static 'bootstrap-table/js/bootstrap-table-zh-CN.min.js' %}"></script>
 <script src="{% static 'sql-formatter/sql-formatter.min.js' %}"></script>
 <script src="{% static 'dist/js/formatter.js' %}"></script>
+<script src="{% static 'dist/js/utils.js' %}"></script>
 
 </body>
 <script>

diff --git a/requirements.txt b/requirements.txt
@@ -26,3 +26,4 @@ uvloop==0.14.0
 httptools==0.1.1
 uvicorn==0.12.2
 pycryptodome==3.10.1
+pyodps==0.10.7.1
diff --git a/sql/engines/__init__.py b/sql/engines/__init__.py
@@ -178,3 +178,7 @@ def get_engine(instance=None):  # pragma: no cover
         from .phoenix import PhoenixEngine
 
         return PhoenixEngine(instance=instance)
+
+    elif instance.db_type == 'odps':
+        from .odps import ODPSEngine
+        return ODPSEngine(instance=instance)
diff --git a/sql/engines/odps.py b/sql/engines/odps.py
@@ -0,0 +1,128 @@
+# -*- coding: UTF-8 -*-
+
+import re
+import logging
+
+from . import EngineBase
+from .models import ResultSet, ReviewSet, ReviewResult
+
+from odps import ODPS
+
+
+logger = logging.getLogger('default')
+
+
+class ODPSEngine(EngineBase):
+
+    def get_connection(self, db_name=None):
+        if self.conn:
+            return self.conn
+
+        db_name = db_name if db_name else self.instance.db_name
+
+        if db_name is None:
+            raise ValueError("db_name不能为空")
+
+        self.conn = ODPS(self.user, self.password, project=db_name, endpoint=self.host)
+
+        return self.conn
+
+    @property
+    def name(self):
+        return 'ODPS'
+
+    @property
+    def info(self):
+        return 'ODPS engine'
+
+    def get_all_databases(self):
+        """获取数据库列表, 返回一个ResultSet
+            ODPS只有project概念, 直接返回project名称
+        """
+        result = ResultSet()
+
+        try:
+            conn = self.get_connection(self.get_connection())
+            result.rows = [conn.project]
+        except Exception as e:
+            logger.warning(f"ODPS执行异常, {e}")
+            result.rows = [self.instance.db_name]
+        return result
+
+    def get_all_tables(self, db_name, **kwargs):
+        """获取table 列表, 返回一个ResultSet"""
+
+        db_name = db_name if db_name else self.instance.db_name
+        result_set = ResultSet()
+
+        try:
+            conn = self.get_connection(db_name=db_name)
+
+            rows = [t.name for t in conn.list_tables()]
+            result_set.rows = rows
+
+        except Exception as e:
+            logger.warning(f"ODPS语句执行报错, 错误信息{e}")
+            result_set.error = str(e)
+
+        return result_set
+
+    def get_all_columns_by_tb(self, db_name, tb_name, **kwargs):
+        """获取所有字段, 返回一个ResultSet"""
+
+        column_list = ['COLUMN_NAME', 'COLUMN_TYPE', 'COLUMN_COMMENT']
+
+        conn = self.get_connection(db_name)
+
+        table = conn.get_table(tb_name)
+
+        schema_cols = table.schema.columns
+
+        rows = []
+
+        for col in schema_cols:
+            rows.append([col.name, str(col.type), col.comment])
+
+        result = ResultSet()
+        result.column_list = column_list
+        result.rows = rows
+        return result
+
+    def describe_table(self, db_name, tb_name, **kwargs):
+        """return ResultSet 类似查询"""
+
+        result = self.get_all_columns_by_tb(db_name, tb_name)
+
+        return result
+
+    def query(self, db_name=None, sql='', limit_num=0, close_conn=True,  **kwargs):
+        """返回 ResultSet """
+        result_set = ResultSet(full_sql=sql)
+
+        if not re.match(r"^select", sql, re.I):
+            result_set.error = str("仅支持ODPS查询语句")
+
+        # 存在limit，替换limit; 不存在，添加limit
+        if re.search('limit', sql):
+            sql = re.sub('limit.+(\d+)', 'limit ' + str(limit_num), sql)
+        else:
+            if sql.strip()[-1] == ';':
+                sql = sql[:-1]
+            sql = sql + ' limit ' + str(limit_num) + ';'
+
+        try:
+            conn = self.get_connection(db_name)
+            effect_row = conn.execute_sql(sql)
+            reader = effect_row.open_reader()
+            rows = [row.values for row in reader]
+            column_list = getattr(reader, '_schema').names
+
+            result_set.column_list = column_list
+            result_set.rows = rows
+            result_set.affected_rows = len(rows)
+
+        except Exception as e:
+            logger.warning(f"ODPS语句执行报错, 语句：{sql}，错误信息{e}")
+            result_set.error = str(e)
+        return result_set
+
diff --git a/sql/models.py b/sql/models.py
@@ -86,6 +86,7 @@ class Meta:
     ('oracle', 'Oracle'),
     ('mongo', 'Mongo'),
     ('phoenix', 'Phoenix'),
+    ('odps', 'ODPS'),
     ('goinception', 'goInception'))
 
 

diff --git a/sql/templates/archive.html b/sql/templates/archive.html
@@ -373,9 +373,7 @@ <h4 class="modal-title text-danger">工单日志</h4>
                 }],
                 onLoadSuccess: function () {
                 },
-                onLoadError: function () {
-                    alert("数据加载失败！请检查接口返回信息和错误日志！");
-                },
+                onLoadError: onLoadErrorCallback,
                 onSearch: function (e) {
                     //传搜索参数给服务器
                     queryParams(e)
@@ -629,9 +627,7 @@ <h4 class="modal-title text-danger">工单日志</h4>
                         }],
                         onLoadSuccess: function () {
                         },
-                        onLoadError: function () {
-                            alert("数据加载失败！请检查接口返回信息和错误日志！");
-                        }
+                        onLoadError: onLoadErrorCallback,
                     });
                 },
                 error: function (XMLHttpRequest, textStatus, errorThrown) {

diff --git a/sql/templates/archivedetail.html b/sql/templates/archivedetail.html
@@ -446,9 +446,7 @@ <h4><b>归档条件</b></h4>
                 }],
                 onLoadSuccess: function () {
                 },
-                onLoadError: function () {
-                    alert("数据加载失败！请检查接口返回信息和错误日志！");
-                }
+                onLoadError: onLoadErrorCallback,
             });
         }
 

diff --git a/sql/templates/audit.html b/sql/templates/audit.html
@@ -158,9 +158,7 @@
                 }],
                 onLoadSuccess: function () {
                 },
-                onLoadError: function () {
-                    alert("数据加载失败！请检查接口返回信息和错误日志！");
-                },
+                onLoadError: onLoadErrorCallback,
                 onSearch: function (e) {
                     //传搜索参数给服务器
                     queryParams(e)

diff --git a/sql/templates/audit_sqlquery.html b/sql/templates/audit_sqlquery.html
@@ -189,9 +189,7 @@
                 }],
                 onLoadSuccess: function () {
                 },
-                onLoadError: function () {
-                    alert("数据加载失败！请检查接口返回信息和错误日志！");
-                },
+                onLoadError: onLoadErrorCallback,
                 onSearch: function (e) {
                     //传搜索参数给服务器
                     queryParams(e)

diff --git a/sql/templates/audit_sqlworkflow.html b/sql/templates/audit_sqlworkflow.html
@@ -248,9 +248,7 @@ <h4 class="modal-title text-danger">工单日志</h4>
                 }],
                 onLoadSuccess: function () {
                 },
-                onLoadError: function () {
-                    alert("数据加载失败！请检查接口返回信息和错误日志！");
-                },
+                onLoadError: onLoadErrorCallback,
                 onSearch: function (e) {
                     //传搜索参数给服务器
                     queryParams(e)
@@ -318,9 +316,7 @@ <h4 class="modal-title text-danger">工单日志</h4>
                         }],
                         onLoadSuccess: function () {
                         },
-                        onLoadError: function () {
-                            alert("数据加载失败！请检查接口返回信息和错误日志！");
-                        }
+                        onLoadError: onLoadErrorCallback,
                     });
                 },
                 error: function (XMLHttpRequest, textStatus, errorThrown) {

diff --git a/sql/templates/binlog2sql.html b/sql/templates/binlog2sql.html
@@ -369,9 +369,7 @@ <h5 class="control-label text-bold" style="color: red">
                                 }],
                                 onLoadSuccess: function () {
                                 },
-                                onLoadError: function () {
-                                    alert("数据加载失败！请检查接口返回信息和错误日志！");
-                                }
+                                onLoadError: onLoadErrorCallback,
                             });
                         } else {
                             alert(data.msg);

diff --git a/sql/templates/database.html b/sql/templates/database.html
@@ -235,9 +235,7 @@ <h4 class="modal-title">编辑/录入数据库信息</h4>
 
                     }
                 },
-                onLoadError: function () {
-                    alert("数据加载失败！请检查接口返回信息和错误日志！");
-                },
+                onLoadError: onLoadErrorCallback,
                 onSearch: function (e) {
                     //传搜索参数给服务器
                     queryParams(e)

diff --git a/sql/templates/dbdiagnostic.html b/sql/templates/dbdiagnostic.html
@@ -196,9 +196,7 @@ <h4 class="modal-title text-danger">确定要终止所选会话吗？</h4>
                             alert("数据加载失败！" + data.msg);
                         }
                     },
-                    onLoadError: function () {
-                        alert("数据加载失败！请检查接口返回信息和错误日志！");
-                    },
+                    onLoadError: onLoadErrorCallback,
                     onSearch: function (e) {
                         //传搜索参数给服务器
                         queryParams(e)
@@ -292,9 +290,7 @@ <h4 class="modal-title text-danger">确定要终止所选会话吗？</h4>
                             alert("数据加载失败！" + data.msg);
                         }
                     },
-                    onLoadError: function () {
-                        alert("数据加载失败！请检查接口返回信息和错误日志！");
-                    },
+                    onLoadError: onLoadErrorCallback,
                     onSearch: function (e) {
                         //传搜索参数给服务器
                         queryParams(e)
@@ -430,9 +426,7 @@ <h4 class="modal-title text-danger">确定要终止所选会话吗？</h4>
                             alert("数据加载失败！" + data.msg);
                         }
                     },
-                    onLoadError: function () {
-                        alert("数据加载失败！请检查接口返回信息和错误日志！");
-                    },
+                    onLoadError: onLoadErrorCallback,
                     onSearch: function (e) {
                         //传搜索参数给服务器
                         queryParams(e)
@@ -538,9 +532,7 @@ <h4 class="modal-title text-danger">确定要终止所选会话吗？</h4>
                             alert("数据加载失败！" + data.msg);
                         }
                     },
-                    onLoadError: function () {
-                        alert("数据加载失败！请检查接口返回信息和错误日志！");
-                    },
+                    onLoadError: onLoadErrorCallback,
                     onSearch: function (e) {
                         //传搜索参数给服务器
                         queryParams(e)

diff --git a/sql/templates/detail.html b/sql/templates/detail.html
@@ -676,9 +676,7 @@ <h4 class="modal-title text-danger">修改可执行时间</h4>
                 },
                 onLoadSuccess: function () {
                 },
-                onLoadError: function () {
-                    alert("数据加载失败！请检查接口返回信息和错误日志！");
-                }
+                onLoadError: onLoadErrorCallback,
             });
         }
 
@@ -734,9 +732,7 @@ <h4 class="modal-title text-danger">修改可执行时间</h4>
                         }],
                         onLoadSuccess: function () {
                         },
-                        onLoadError: function () {
-                            alert("数据加载失败！请检查接口返回信息和错误日志！");
-                        }
+                        onLoadError: onLoadErrorCallback,
                     });
                 },
                 error: function (XMLHttpRequest, textStatus, errorThrown) {
@@ -839,9 +835,7 @@ <h4 class="modal-title text-danger">修改可执行时间</h4>
                     $("#osc-toolbar").show();
                     $('#osc_percent').modal('show');
                 },
-                onLoadError: function () {
-                    alert("数据加载失败！请检查接口返回信息和错误日志！");
-                }
+                onLoadError: onLoadErrorCallback,
             });
 
         }

diff --git a/sql/templates/group.html b/sql/templates/group.html
@@ -85,9 +85,7 @@
                 }],
                 onLoadSuccess: function () {
                 },
-                onLoadError: function () {
-                    alert("数据加载失败！请检查接口返回信息和错误日志！");
-                },
+                onLoadError: fonLoadErrorCallback,
                 onSearch: function (e) {
                     //传搜索参数给服务器
                     queryParams(e)

diff --git a/sql/templates/groupmgmt.html b/sql/templates/groupmgmt.html
@@ -153,9 +153,7 @@ <h4 class="modal-title" id="myModalLabel">新增关联对象</h4>
                 ],
                 onLoadSuccess: function () {
                 },
-                onLoadError: function () {
-                    alert("数据加载失败！请检查接口返回信息和错误日志！");
-                },
+                onLoadError: onLoadErrorCallback,
                 onSearch: function (e) {
                     //传搜索参数给服务器
                     queryParams(e)

diff --git a/sql/templates/instance.html b/sql/templates/instance.html
@@ -20,6 +20,7 @@
                 <option value="oracle">Oracle</option>
                 <option value="mongo">Mongo</option>
                 <option value="phoenix">Phoenix</option>
+                <option value="odps">ODPS</option>
             </select>
         </div>
         <div class="form-group">
@@ -171,9 +172,7 @@ <h4 class="modal-title" id="myModalLabel">清理Binlog日志</h4>
                     }],
                 onLoadSuccess: function () {
                 },
-                onLoadError: function () {
-                    alert("数据加载失败！请检查接口返回信息和错误日志！");
-                },
+                onLoadError: onLoadErrorCallback,
                 onSearch: function (e) {
                     //传搜索参数给服务器
                     queryParams(e)

diff --git a/sql/templates/instanceaccount.html b/sql/templates/instanceaccount.html
@@ -1152,9 +1152,7 @@ <h4 class="modal-title">删除账号
 
                     }
                 },
-                onLoadError: function () {
-                    alert("数据加载失败！请检查接口返回信息和错误日志！");
-                },
+                onLoadError: onLoadErrorCallback,
                 onSearch: function (e) {
                     //传搜索参数给服务器
                     queryParams(e)