抽象调用命令行工具，做成plugin的形式 fix hhyo#59

sql/plugins/plugin.py

@@ -0,0 +1,66 @@
+# -*- coding: UTF-8 -*-
+""" 
+@author: hhyo 
+@license: Apache Licence 
+@file: plugin.py 
+@time: 2019/03/04
+"""
+__author__ = 'hhyo'
+
+import logging
+import subprocess
+import traceback
+
+logger = logging.getLogger('default')
+
+
+class Plugin:
+    def __init__(self, path):
+        self.path = path
+        self.required_args = []  # 必须参数
+        self.disable_args = []  # 禁用参数
+
+    def check_args(self, args):
+        """
+        检查请求参数列表
+        :return: {'status': 0, 'msg': 'ok', 'data': {}}
+        """
+        args_check_result = {'status': 0, 'msg': 'ok', 'data': {}}
+        # 检查路径
+        if self.path is None:
+            return {'status': 1, 'msg': '可执行文件路径不能为空！', 'data': {}}
+        # 检查禁用参数
+        for arg in args.keys():
+            if arg in self.disable_args:
+                return {'status': 1, 'msg': '{arg}参数已被禁用'.format(arg=arg), 'data': {}}
+        # 检查必须参数
+        for req_arg in self.required_args:
+            if req_arg not in args.keys():
+                return {'status': 1, 'msg': '必须指定{arg}参数'.format(arg=req_arg), 'data': {}}
+            elif args[req_arg] is None or args[req_arg] == '':
+                return {'status': 1, 'msg': '{arg}参数值不能为空'.format(arg=req_arg), 'data': {}}
+        return args_check_result
+
+    def generate_args2cmd(self, args, shell):
+        """
+        将请求参数转换为命令行参数
+        :return:
+        """
+
+    @staticmethod
+    def execute_cmd(cmd_args, shell):
+        """
+        执行命令并且返回执行信息
+        :return:
+        """
+        try:
+            p = subprocess.Popen(cmd_args,
+                                 shell=shell,
+                                 stdout=subprocess.PIPE,
+                                 stderr=subprocess.STDOUT,
+                                 universal_newlines=True)
+            stdout, stderr = p.communicate()
+            return stdout
+        except Exception as e:
+            logger.error("命令执行失败\n{}".format(traceback.format_exc()))
+            raise RuntimeError('命令执行失败，失败原因:%s' % str(e))

sql/plugins/soar.py

@@ -0,0 +1,41 @@
+# -*- coding: UTF-8 -*-
+""" 
+@author: hhyo
+@license: Apache Licence 
+@file: soar.py 
+@time: 2019/03/04
+"""
+__author__ = 'hhyo'
+
+from common.config import SysConfig
+from sql.plugins.plugin import Plugin
+
+
+class Soar(Plugin):
+
+    def __init__(self):
+        self.path = SysConfig().get('soar')
+        self.required_args = ['query']
+        self.disable_args = []
+        super(Plugin, self).__init__()
+
+    def generate_args2cmd(self, args, shell):
+        """
+        转换请求参数为命令行
+        :param args:
+        :param shell:
+        :return:
+        """
+        if shell:
+            cmd_args = self.path if self.path else ''
+            for name, value in args.items():
+                if name in ['query', 'online-dsn', 'test-dsn']:
+                    cmd_args = cmd_args + ' ' + '-{name}="{value}"'.format(name=name, value=str(value))
+                else:
+                    cmd_args = cmd_args + ' ' + '-{name}={value}'.format(name=name, value=str(value))
+        else:
+            cmd_args = [self.path]
+            for name, value in args.items():
+                cmd_args.append('-%s' % name)
+                cmd_args.append('%s' % value)
+        return cmd_args

sql/plugins/sqladvisor.py

@@ -0,0 +1,40 @@
+# -*- coding: UTF-8 -*-
+""" 
+@author: hhyo 
+@license: Apache Licence 
+@file: sqladvisor.py 
+@time: 2019/03/04
+"""
+__author__ = 'hhyo'
+
+from common.config import SysConfig
+from sql.plugins.plugin import Plugin
+
+
+class SQLAdvisor(Plugin):
+    def __init__(self):
+        self.path = SysConfig().get('sqladvisor')
+        self.required_args = ['q']
+        self.disable_args = []
+        super(Plugin, self).__init__()
+
+    def generate_args2cmd(self, args, shell):
+        """
+        转换请求参数为命令行
+        :param args:
+        :param shell:
+        :return:
+        """
+        if shell:
+            cmd_args = self.path if self.path else ''
+            for name, value in args.items():
+                if name == 'v':
+                    cmd_args = cmd_args + ' ' + '-{name} {value}'.format(name=name, value=str(value))
+                else:
+                    cmd_args = cmd_args + ' ' + '-{name} "{value}"'.format(name=name, value=str(value))
+        else:
+            cmd_args = [self.path]
+            for name, value in args.items():
+                cmd_args.append('-%s' % name)
+                cmd_args.append('%s' % value)
+        return cmd_args

sql/plugins/tests.py

@@ -0,0 +1,146 @@
+# -*- coding: UTF-8 -*-
+""" 
+@author: hhyo 
+@license: Apache Licence 
+@file: tests.py 
+@time: 2019/03/04
+"""
+import json
+
+from django.test import Client, TestCase
+from django.contrib.auth import get_user_model
+from sql.plugins.soar import Soar
+from sql.plugins.sqladvisor import SQLAdvisor
+
+from common.config import SysConfig
+
+User = get_user_model()
+
+__author__ = 'hhyo'
+
+
+class TestPlugin(TestCase):
+    """
+    测试Plugin调用
+    """
+
+    def setUp(self):
+        self.superuser = User(username='super', is_superuser=True)
+        self.superuser.save()
+        self.sys_config = SysConfig()
+        self.client = Client()
+        self.client.force_login(self.superuser)
+
+    def tearDown(self):
+        self.superuser.delete()
+        self.sys_config.replace(json.dumps({}))
+
+    def test_check_args_path(self):
+        """
+        测试路径
+        :return:
+        """
+        args = {"online-dsn": '',
+                "test-dsn": '',
+                "allow-online-as-test": "false",
+                "report-type": "markdown",
+                "query": "select 1;"
+                }
+        self.sys_config.set('soar', '')
+        self.sys_config.get_all_config()
+        soar = Soar()
+        args_check_result = soar.check_args(args)
+        self.assertDictEqual(args_check_result, {'status': 1, 'msg': '可执行文件路径不能为空！', 'data': {}})
+        # 路径不为空
+        self.sys_config.set('soar', '/opt/archery/src/plugins/soar')
+        self.sys_config.get_all_config()
+        soar = Soar()
+        args_check_result = soar.check_args(args)
+        self.assertDictEqual(args_check_result, {'status': 0, 'msg': 'ok', 'data': {}})
+
+    def test_check_args_disable(self):
+        """
+        测试禁用参数
+        :return:
+        """
+        args = {"online-dsn": '',
+                "test-dsn": '',
+                "allow-online-as-test": "false",
+                "report-type": "markdown",
+                "query": "select 1;"
+                }
+        self.sys_config.set('soar', '/opt/archery/src/plugins/soar')
+        self.sys_config.get_all_config()
+        soar = Soar()
+        soar.disable_args = ['allow-online-as-test']
+        args_check_result = soar.check_args(args)
+        self.assertDictEqual(args_check_result, {'status': 1, 'msg': 'allow-online-as-test参数已被禁用', 'data': {}})
+
+    def test_check_args_required(self):
+        """
+        测试必选参数
+        :return:
+        """
+        args = {"online-dsn": '',
+                "test-dsn": '',
+                "allow-online-as-test": "false",
+                "report-type": "markdown",
+                }
+        self.sys_config.set('soar', '/opt/archery/src/plugins/soar')
+        self.sys_config.get_all_config()
+        soar = Soar()
+        soar.required_args = ['query']
+        args_check_result = soar.check_args(args)
+        self.assertDictEqual(args_check_result, {'status': 1, 'msg': '必须指定query参数', 'data': {}})
+        args['query'] = ""
+        args_check_result = soar.check_args(args)
+        self.assertDictEqual(args_check_result, {'status': 1, 'msg': 'query参数值不能为空', 'data': {}})
+
+    def test_soar_generate_args2cmd(self):
+        args = {"online-dsn": '',
+                "test-dsn": '',
+                "allow-online-as-test": "false",
+                "report-type": "markdown",
+                "query": "select 1;"
+                }
+        self.sys_config.set('soar', '/opt/archery/src/plugins/soar')
+        self.sys_config.get_all_config()
+        soar = Soar()
+        cmd_args = soar.generate_args2cmd(args, False)
+        self.assertIsInstance(cmd_args, list)
+        cmd_args = soar.generate_args2cmd(args, True)
+        self.assertIsInstance(cmd_args, str)
+
+    def test_sql_advisor_generate_args2cmd(self):
+        args = {"h": 'mysql',
+                "P": 3306,
+                "u": 'root',
+                "p": '',
+                "d": 'archery',
+                "v": 1,
+                "q": 'select 1;'
+                }
+        self.sys_config.set('sqladvisor', '/opt/archery/src/plugins/SQLAdvisor')
+        self.sys_config.get_all_config()
+        sql_advisor = SQLAdvisor()
+        cmd_args = sql_advisor.generate_args2cmd(args, False)
+        self.assertIsInstance(cmd_args, list)
+        cmd_args = sql_advisor.generate_args2cmd(args, True)
+        self.assertIsInstance(cmd_args, str)
+
+    def test_execute_cmd(self):
+        args = {"online-dsn": '',
+                "test-dsn": '',
+                "allow-online-as-test": "false",
+                "report-type": "markdown",
+                "query": "select 1;"
+                }
+        self.sys_config.set('soar', '/opt/archery/src/plugins/soar')
+        self.sys_config.get_all_config()
+        soar = Soar()
+        cmd_args = soar.generate_args2cmd(args, True)
+        result = soar.execute_cmd(cmd_args, True)
+        self.assertEqual(result, '/bin/sh: /opt/archery/src/plugins/soar: No such file or directory\n')
+        # 异常
+        with self.assertRaises(RuntimeError):
+            soar.execute_cmd(cmd_args, False)

sql/soar.py → sql/soar_web.py

@@ -1,31 +1,16 @@
 # -*- coding:utf-8 -*-
-import logging
-import subprocess
-import traceback
-
 import simplejson as json
 from django.contrib.auth.decorators import permission_required
-from django.contrib.auth.mixins import PermissionRequiredMixin
 from django.http import HttpResponse
 from common.config import SysConfig
 from sql.models import Instance
 from sql.utils.resource_group import user_instances
-
-logger = logging.getLogger('default')
-
-
-class Soar(PermissionRequiredMixin):
-    permission_required = 'sql.optimize_soar'
-    raise_exception = True
-
-    def __init__(self):
-        self.soar_path = SysConfig().get('soar')
-        self.soar_test_dsn = SysConfig().get('soar_test_dsn')
+from sql.plugins.soar import Soar
 
 
 # 获取soar的处理结果
 @permission_required('sql.optimize_soar', raise_exception=True)
-def soar(request):
+def optimize_soar(request):
     instance_name = request.POST.get('instance_name')
     db_name = request.POST.get('db_name')
     sql = request.POST.get('sql')
@@ -43,33 +28,41 @@ def soar(request):
         result['msg'] = '你所在组未关联该实例'
         return HttpResponse(json.dumps(result), content_type='application/json')
 
-    sql = sql.strip().replace('"', '\\"').replace('`', '').replace('\n', ' ')
+    # 检查测试实例的连接信息和soar程序路径
+    soar_test_dsn = SysConfig().get('soar_test_dsn')
+    soar_path = SysConfig().get('soar')
+    if not (soar_path and soar_test_dsn):
+        result['status'] = 1
+        result['msg'] = '请配置soar_path和test_dsn！'
+        return HttpResponse(json.dumps(result), content_type='application/json')
+
     # 目标实例的连接信息
     instance_info = Instance.objects.get(instance_name=instance_name)
     online_dsn = "{user}:{pwd}@{host}:{port}/{db}".format(user=instance_info.user,
                                                           pwd=instance_info.raw_password,
                                                           host=instance_info.host,
                                                           port=instance_info.port,
                                                           db=db_name)
-    # 获取测试实例的连接信息和soar程序路径
-    soar_cfg = Soar()
-    test_dsn = soar_cfg.soar_test_dsn
-    soar_path = soar_cfg.soar_path
-    if not (soar_path and test_dsn):
-        result['status'] = 1
-        result['msg'] = '请配置soar_path和test_dsn！'
-        return HttpResponse(json.dumps(result), content_type='application/json')
 
     # 提交给soar获取分析报告
+    soar = Soar()
+    # 准备参数
+    args = {"online-dsn": online_dsn,
+            "test-dsn": soar_test_dsn,
+            "allow-online-as-test": "false",
+            "report-type": "markdown",
+            "query": sql.strip().replace('"', '\\"').replace('`', '').replace('\n', ' ')
+            }
+    # 参数检查
+    args_check_result = soar.check_args(args)
+    if args_check_result['status'] == 1:
+        return HttpResponse(json.dumps(args_check_result), content_type='application/json')
+    # 参数转换
+    cmd_args = soar.generate_args2cmd(args, shell=True)
+    # 执行命令
     try:
-        p = subprocess.Popen(
-            soar_path + ' -allow-online-as-test=false -report-type=markdown' +
-            ' -query "{}" -online-dsn "{}" -test-dsn "{}" '.format(sql.strip(), online_dsn, test_dsn),
-            stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True,
-            universal_newlines=True)
-        stdout, stderr = p.communicate()
-        result['data'] = stdout
-    except Exception:
-        logger.error(traceback.format_exc())
-        result['data'] = 'soar运行报错，请检查相关日志'
+        result['data'] = soar.execute_cmd(cmd_args, shell=True)
+    except RuntimeError as e:
+        result['status'] = 1
+        result['msg'] = str(e)
     return HttpResponse(json.dumps(result), content_type='application/json')