From 24989535f11195178c3b227b3af6803eae27b6b4 Mon Sep 17 00:00:00 2001 From: nick2wang Date: Fri, 18 Feb 2022 17:33:39 +0800 Subject: [PATCH 1/4] =?UTF-8?q?=E6=94=AF=E6=8C=81redis=E9=9B=86=E7=BE=A4?= =?UTF-8?q?=E6=A8=A1=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 支持redis集群模式 --- common/static/dist/js/utils.js | 18 ++++++++++++++++++ sql/admin.py | 3 ++- sql/engines/__init__.py | 1 + sql/engines/redis.py | 11 ++++++++--- sql/form.py | 8 +++++++- sql/models.py | 1 + src/init_sql/v1.8.3.sql | 5 ++++- 7 files changed, 41 insertions(+), 6 deletions(-) diff --git a/common/static/dist/js/utils.js b/common/static/dist/js/utils.js index b706c69cc9..4ff8392310 100644 --- a/common/static/dist/js/utils.js +++ b/common/static/dist/js/utils.js @@ -5,3 +5,21 @@ var onLoadErrorCallback = function (status, jqXHR) { alert("未知错误，请联系管理员处理！"); } }; + +// 实例配置页面根据db_type选择显示或隐藏mode字段，mode字段只适用于redis实例 +(function($) { + $(function() { + let db_type = $('#id_db_type'); + let mode = $('#id_mode').parent().parent(); + + function toggleMode(value) { + value === 'redis' ? mode.show() : mode.hide(); + } + + toggleMode(db_type.val()); + + db_type.change(function() { + toggleMode($(this).val()); + }); + }); +})(django.jQuery); diff --git a/sql/admin.py b/sql/admin.py index 2389956f10..d3f532caad 100755 --- a/sql/admin.py +++ b/sql/admin.py @@ -11,7 +11,7 @@ WorkflowAudit, WorkflowLog, ParamTemplate, ParamHistory, InstanceTag, \ Tunnel, AuditEntry -from sql.form import TunnelForm +from sql.form import TunnelForm, InstanceForm # 用户管理 @@ -62,6 +62,7 @@ def get_readonly_fields(self, request, obj=None): # 实例管理 @admin.register(Instance) class InstanceAdmin(admin.ModelAdmin): + form = InstanceForm list_display = ('id', 'instance_name', 'db_type', 'type', 'host', 'port', 'user', 'create_time') search_fields = ['instance_name', 'host', 'port', 'user'] list_filter = ('db_type', 'type', 'instance_tag') diff --git a/sql/engines/__init__.py b/sql/engines/__init__.py index 00d91f3d91..26aef9bffe 100644 --- a/sql/engines/__init__.py +++ b/sql/engines/__init__.py @@ -17,6 +17,7 @@ def __init__(self, instance=None): self.user = instance.user self.password = instance.password self.db_name = instance.db_name + self.mode = instance.mode # 判断如果配置了隧道则连接隧道，只测试了MySQL if self.instance.tunnel: diff --git a/sql/engines/redis.py b/sql/engines/redis.py index 99b90a94bb..9e01a553a7 100644 --- a/sql/engines/redis.py +++ b/sql/engines/redis.py @@ -9,7 +9,8 @@ import re import shlex -import redis +from redis import Redis +from redis.cluster import RedisCluster import logging import traceback @@ -25,8 +26,12 @@ class RedisEngine(EngineBase): def get_connection(self, db_name=None): db_name = db_name or self.db_name - return redis.Redis(host=self.host, port=self.port, db=db_name, password=self.password, - encoding_errors='ignore', decode_responses=True, socket_connect_timeout=10) + if self.mode == 'cluster': + return RedisCluster(host=self.host, port=self.port, password=self.password, + encoding_errors='ignore', decode_responses=True, socket_connect_timeout=10) + else: + return Redis(host=self.host, port=self.port, db=db_name, password=self.password, + encoding_errors='ignore', decode_responses=True, socket_connect_timeout=10) @property def name(self): diff --git a/sql/form.py b/sql/form.py index 9b8280bb37..7459dd9f6f 100644 --- a/sql/form.py +++ b/sql/form.py @@ -9,7 +9,7 @@ --------------------------------------------------------- """ from django.forms import ModelForm, Textarea -from sql.models import Tunnel +from sql.models import Tunnel, Instance from django.core.exceptions import ValidationError @@ -30,3 +30,9 @@ def clean(self): cleaned_data['pkey'] = str(pkey_path, 'utf-8').replace(r'\r', '').replace(r'\n', '') except IOError: raise ValidationError("秘钥文件不存在， 请勾选秘钥路径的清除选项再进行保存") + + +class InstanceForm(ModelForm): + class Media: + model = Instance + js = ('jquery/jquery.min.js', 'dist/js/utils.js', ) diff --git a/sql/models.py b/sql/models.py index 883557d9c1..a829cb51d2 100755 --- a/sql/models.py +++ b/sql/models.py @@ -129,6 +129,7 @@ class Instance(models.Model): instance_name = models.CharField('实例名称', max_length=50, unique=True) type = models.CharField('实例类型', max_length=6, choices=(('master', '主库'), ('slave', '从库'))) db_type = models.CharField('数据库类型', max_length=20, choices=DB_TYPE_CHOICES) + mode = models.CharField('运行模式', max_length=10, default='', choices=(('standalone', '单机'), ('cluster', '集群'))) host = models.CharField('实例连接', max_length=200) port = models.IntegerField('端口', default=0) user = fields.EncryptedCharField(verbose_name='用户名', max_length=200, default='', blank=True) diff --git a/src/init_sql/v1.8.3.sql b/src/init_sql/v1.8.3.sql index 732810c819..bc04cb4942 100644 --- a/src/init_sql/v1.8.3.sql +++ b/src/init_sql/v1.8.3.sql @@ -28,4 +28,7 @@ insert IGNORE INTO auth_permission (name, content_type_id, codename) VALUES -- 在线查询下载权限 set @content_type_id=(select id from django_content_type where app_label='sql' and model='permission'); insert IGNORE INTO auth_permission (name, content_type_id, codename) VALUES -('在线查询下载权限', @content_type_id, 'query_download'); \ No newline at end of file +('在线查询下载权限', @content_type_id, 'query_download'); + +-- 实例配置表新增mode字段，用于redis实例 +alter table sql_instance add column `mode` varchar(10) DEFAULT '' after `db_type`; From bba3a0c129a0298dda9c43dae1b33698f9a282fe Mon Sep 17 00:00:00 2001 From: nick2wang Date: Fri, 18 Feb 2022 17:58:54 +0800 Subject: [PATCH 2/4] =?UTF-8?q?=E5=8D=95=E5=85=83=E6=B5=8B=E8=AF=95?= =?UTF-8?q?=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 单元测试更新 --- sql/engines/tests.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/sql/engines/tests.py b/sql/engines/tests.py index c5413fd208..1f6fc6ec2f 100644 --- a/sql/engines/tests.py +++ b/sql/engines/tests.py @@ -539,8 +539,8 @@ def test_seconds_behind_master(self, _query): class TestRedis(TestCase): @classmethod def setUpClass(cls): - cls.ins = Instance(instance_name='some_ins', type='slave', db_type='redis', host='some_host', - port=1366, user='ins_user', password='some_str') + cls.ins = Instance(instance_name='some_ins', type='slave', db_type='redis', mode='standalone', + host='some_host', port=1366, user='ins_user', password='some_str') cls.ins.save() @classmethod @@ -549,40 +549,40 @@ def tearDownClass(cls): SqlWorkflow.objects.all().delete() SqlWorkflowContent.objects.all().delete() - @patch('redis.Redis') + @patch('Redis') def test_engine_base_info(self, _conn): new_engine = RedisEngine(instance=self.ins) self.assertEqual(new_engine.name, 'Redis') self.assertEqual(new_engine.info, 'Redis engine') - @patch('redis.Redis') + @patch('Redis') def test_get_connection(self, _conn): new_engine = RedisEngine(instance=self.ins) new_engine.get_connection() _conn.assert_called_once() - @patch('redis.Redis.execute_command', return_value=[1, 2, 3]) + @patch('Redis.execute_command', return_value=[1, 2, 3]) def test_query_return_list(self, _execute_command): new_engine = RedisEngine(instance=self.ins) query_result = new_engine.query(db_name=0, sql='keys *', limit_num=100) self.assertIsInstance(query_result, ResultSet) self.assertTupleEqual(query_result.rows, ([1], [2], [3])) - @patch('redis.Redis.execute_command', return_value='text') + @patch('Redis.execute_command', return_value='text') def test_query_return_str(self, _execute_command): new_engine = RedisEngine(instance=self.ins) query_result = new_engine.query(db_name=0, sql='keys *', limit_num=100) self.assertIsInstance(query_result, ResultSet) self.assertTupleEqual(query_result.rows, (['text'],)) - @patch('redis.Redis.execute_command', return_value='text') + @patch('Redis.execute_command', return_value='text') def test_query_execute(self, _execute_command): new_engine = RedisEngine(instance=self.ins) query_result = new_engine.query(db_name=0, sql='keys *', limit_num=100) self.assertIsInstance(query_result, ResultSet) self.assertTupleEqual(query_result.rows, (['text'],)) - @patch('redis.Redis.config_get', return_value={"databases": 4}) + @patch('Redis.config_get', return_value={"databases": 4}) def test_get_all_databases(self, _config_get): new_engine = RedisEngine(instance=self.ins) dbs = new_engine.get_all_databases() @@ -628,7 +628,7 @@ def test_execute_check(self): self.assertIsInstance(check_result, ReviewSet) self.assertEqual(check_result.rows[0].__dict__, row.__dict__) - @patch('redis.Redis.execute_command', return_value='text') + @patch('Redis.execute_command', return_value='text') def test_execute_workflow_success(self, _execute_command): sql = 'set 1 1' row = ReviewResult(id=1, From b915a7a41fc726262c6c59ae837657357f0a8f51 Mon Sep 17 00:00:00 2001 From: nick2wang Date: Fri, 18 Feb 2022 18:09:18 +0800 Subject: [PATCH 3/4] =?UTF-8?q?=E5=8D=95=E5=85=83=E6=B5=8B=E8=AF=95?= =?UTF-8?q?=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 单元测试更新 --- sql/engines/redis.py | 7 +++---- sql/engines/tests.py | 14 +++++++------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/sql/engines/redis.py b/sql/engines/redis.py index 9e01a553a7..564431d140 100644 --- a/sql/engines/redis.py +++ b/sql/engines/redis.py @@ -9,8 +9,7 @@ import re import shlex -from redis import Redis -from redis.cluster import RedisCluster +import redis import logging import traceback @@ -27,10 +26,10 @@ class RedisEngine(EngineBase): def get_connection(self, db_name=None): db_name = db_name or self.db_name if self.mode == 'cluster': - return RedisCluster(host=self.host, port=self.port, password=self.password, + return redis.cluster.RedisCluster(host=self.host, port=self.port, password=self.password, encoding_errors='ignore', decode_responses=True, socket_connect_timeout=10) else: - return Redis(host=self.host, port=self.port, db=db_name, password=self.password, + return redis.Redis(host=self.host, port=self.port, db=db_name, password=self.password, encoding_errors='ignore', decode_responses=True, socket_connect_timeout=10) @property diff --git a/sql/engines/tests.py b/sql/engines/tests.py index 1f6fc6ec2f..159fdefc7e 100644 --- a/sql/engines/tests.py +++ b/sql/engines/tests.py @@ -549,40 +549,40 @@ def tearDownClass(cls): SqlWorkflow.objects.all().delete() SqlWorkflowContent.objects.all().delete() - @patch('Redis') + @patch('redis.Redis') def test_engine_base_info(self, _conn): new_engine = RedisEngine(instance=self.ins) self.assertEqual(new_engine.name, 'Redis') self.assertEqual(new_engine.info, 'Redis engine') - @patch('Redis') + @patch('redis.Redis') def test_get_connection(self, _conn): new_engine = RedisEngine(instance=self.ins) new_engine.get_connection() _conn.assert_called_once() - @patch('Redis.execute_command', return_value=[1, 2, 3]) + @patch('redis.Redis.execute_command', return_value=[1, 2, 3]) def test_query_return_list(self, _execute_command): new_engine = RedisEngine(instance=self.ins) query_result = new_engine.query(db_name=0, sql='keys *', limit_num=100) self.assertIsInstance(query_result, ResultSet) self.assertTupleEqual(query_result.rows, ([1], [2], [3])) - @patch('Redis.execute_command', return_value='text') + @patch('redis.Redis.execute_command', return_value='text') def test_query_return_str(self, _execute_command): new_engine = RedisEngine(instance=self.ins) query_result = new_engine.query(db_name=0, sql='keys *', limit_num=100) self.assertIsInstance(query_result, ResultSet) self.assertTupleEqual(query_result.rows, (['text'],)) - @patch('Redis.execute_command', return_value='text') + @patch('redis.Redis.execute_command', return_value='text') def test_query_execute(self, _execute_command): new_engine = RedisEngine(instance=self.ins) query_result = new_engine.query(db_name=0, sql='keys *', limit_num=100) self.assertIsInstance(query_result, ResultSet) self.assertTupleEqual(query_result.rows, (['text'],)) - @patch('Redis.config_get', return_value={"databases": 4}) + @patch('redis.Redis.config_get', return_value={"databases": 4}) def test_get_all_databases(self, _config_get): new_engine = RedisEngine(instance=self.ins) dbs = new_engine.get_all_databases() @@ -628,7 +628,7 @@ def test_execute_check(self): self.assertIsInstance(check_result, ReviewSet) self.assertEqual(check_result.rows[0].__dict__, row.__dict__) - @patch('Redis.execute_command', return_value='text') + @patch('redis.Redis.execute_command', return_value='text') def test_execute_workflow_success(self, _execute_command): sql = 'set 1 1' row = ReviewResult(id=1, From 002db819ad208a626ebece260ee11c753b4fd8bd Mon Sep 17 00:00:00 2001 From: nick2wang Date: Sat, 19 Feb 2022 00:39:28 +0800 Subject: [PATCH 4/4] =?UTF-8?q?=E5=8E=86=E5=8F=B2=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E8=AE=BE=E7=BD=AE=E9=BB=98=E8=AE=A4=E5=80=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 历史数据设置默认值 --- sql/models.py | 2 +- src/init_sql/v1.8.3.sql | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/sql/models.py b/sql/models.py index a829cb51d2..3fa06286bc 100755 --- a/sql/models.py +++ b/sql/models.py @@ -129,7 +129,7 @@ class Instance(models.Model): instance_name = models.CharField('实例名称', max_length=50, unique=True) type = models.CharField('实例类型', max_length=6, choices=(('master', '主库'), ('slave', '从库'))) db_type = models.CharField('数据库类型', max_length=20, choices=DB_TYPE_CHOICES) - mode = models.CharField('运行模式', max_length=10, default='', choices=(('standalone', '单机'), ('cluster', '集群'))) + mode = models.CharField('运行模式', max_length=10, default='', blank=True, choices=(('standalone', '单机'), ('cluster', '集群'))) host = models.CharField('实例连接', max_length=200) port = models.IntegerField('端口', default=0) user = fields.EncryptedCharField(verbose_name='用户名', max_length=200, default='', blank=True) diff --git a/src/init_sql/v1.8.3.sql b/src/init_sql/v1.8.3.sql index bc04cb4942..17b07bc25c 100644 --- a/src/init_sql/v1.8.3.sql +++ b/src/init_sql/v1.8.3.sql @@ -30,5 +30,6 @@ set @content_type_id=(select id from django_content_type where app_label='sql' a insert IGNORE INTO auth_permission (name, content_type_id, codename) VALUES ('在线查询下载权限', @content_type_id, 'query_download'); --- 实例配置表新增mode字段，用于redis实例 +-- 实例配置表新增mode字段，用于redis实例；为历史数据设置默认值 alter table sql_instance add column `mode` varchar(10) DEFAULT '' after `db_type`; +update sql_instance set mode='standalone' where db_type='redis';