From a5b3eb00afd5ab9cbde436ef44428dedd9919429 Mon Sep 17 00:00:00 2001 From: Aleksandr Kuzminsky Date: Fri, 2 Dec 2016 11:18:05 -0800 Subject: [PATCH 1/3] Fix #6 --- Makefile | 7 --- requirements_dev.txt | 1 + tests/unit/test_twindb_table_compare.py | 39 ++++++++++++++-- tox.ini | 7 ++- twindb_table_compare/cli.py | 1 + twindb_table_compare/clogging.py | 1 + twindb_table_compare/twindb_table_compare.py | 48 +++++++++++++++----- vagrant/Vagrantfile | 14 ++++-- 8 files changed, 90 insertions(+), 28 deletions(-) diff --git a/Makefile b/Makefile index 1cf1885..bd6e1c0 100644 --- a/Makefile +++ b/Makefile @@ -63,13 +63,6 @@ test: lint vagrant-up vagrant-provision ## run tests quickly with the default Py test-all: ## run tests on every Python version with tox tox -coverage: ## check code coverage quickly with the default Python - coverage run --source twindb_table_compare py.test - - coverage report -m - coverage html - $(BROWSER) htmlcov/index.html - docs: ## generate Sphinx HTML documentation, including API docs rm -f docs/twindb_table_compare.rst rm -f docs/modules.rst diff --git a/requirements_dev.txt b/requirements_dev.txt index 010e221..c582a2e 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -10,3 +10,4 @@ Sphinx==1.4.5 PyYAML==3.12 pytest==3.0.2 mysql +pytest-cov diff --git a/tests/unit/test_twindb_table_compare.py b/tests/unit/test_twindb_table_compare.py index 788240d..5b68bc6 100644 --- a/tests/unit/test_twindb_table_compare.py +++ b/tests/unit/test_twindb_table_compare.py @@ -13,17 +13,15 @@ from click.testing import CliRunner from twindb_table_compare import cli -from twindb_table_compare.twindb_table_compare import is_printable +from twindb_table_compare.twindb_table_compare import is_printable, diff def test_command_line_interface(): runner = CliRunner() result = runner.invoke(cli.main) assert result.exit_code == 0 - # assert 'twindb_table_compare.cli.main' in result.output help_result = runner.invoke(cli.main, ['--help']) assert help_result.exit_code == 0 - # assert '--help Show this message and exit.' in help_result.output @pytest.mark.parametrize('input_str,result', [ @@ -38,3 +36,38 @@ def test_command_line_interface(): ]) def test_is_printable(input_str, result): assert is_printable(input_str) == result + + +@pytest.mark.parametrize('master_lines, slave_lines, difference', [ + ( + [ + 'localhost\troot\t1\t2016-12-02 04:46:12\n', + 'master.box\troot\t1\t2016-12-02 04:46:12\n' + ], + [ + 'localhost\troot\t1\t2016-12-02 05:43:47\n', + 'slave.box\troot\t1\t2016-12-02 05:43:47\n' + ], + """@@ -1,2 +1,2 @@ +-localhost\troot\t1\t2016-12-02 04:46:12 +-master.box\troot\t1\t2016-12-02 04:46:12 ++localhost\troot\t1\t2016-12-02 05:43:47 ++slave.box\troot\t1\t2016-12-02 05:43:47 +""" + ), + ( + [ + '3882\t2016-04-20 14:57:31\n', + '3882\t2016-04-20 14:57:31\n', + '3937\t2016-05-13 14:32:53\n' + ], + [ + '3937\t2016-05-13 14:32:53\n', + '3882\t2016-04-20 14:57:31\n', + '3882\t2016-04-20 14:57:31\n' + ], + "" + ) +]) +def test_diff(master_lines, slave_lines, difference): + assert diff(master_lines, slave_lines) == difference diff --git a/tox.ini b/tox.ini index 9879b1e..d6d4c5b 100644 --- a/tox.ini +++ b/tox.ini @@ -1,12 +1,15 @@ [tox] -#envlist = py26, py27, py34, py35, flake8 -envlist = py26, py27, flake8 +envlist = py26, py27, flake8, cov [testenv:flake8] basepython=python deps=flake8 commands=flake8 twindb_table_compare +[testenv:cov] +deps=-rrequirements_dev.txt +commands=py.test --cov=twindb_table_compare --cov-report term-missing tests/unit + [testenv] setenv = PYTHONPATH = {toxinidir}:{toxinidir}/twindb_table_compare diff --git a/twindb_table_compare/cli.py b/twindb_table_compare/cli.py index 0c44115..c2aed8a 100644 --- a/twindb_table_compare/cli.py +++ b/twindb_table_compare/cli.py @@ -29,5 +29,6 @@ def main(user, password, db, tbl, slave): password, ch_db=db, ch_tbl=tbl) + if __name__ == "__main__": main() diff --git a/twindb_table_compare/clogging.py b/twindb_table_compare/clogging.py index f8a9165..1b2b2d4 100644 --- a/twindb_table_compare/clogging.py +++ b/twindb_table_compare/clogging.py @@ -150,5 +150,6 @@ def main(): logging.error('ERROR') logging.critical('CRITICAL') + if __name__ == '__main__': main() diff --git a/twindb_table_compare/twindb_table_compare.py b/twindb_table_compare/twindb_table_compare.py index d4e62ff..de7358d 100644 --- a/twindb_table_compare/twindb_table_compare.py +++ b/twindb_table_compare/twindb_table_compare.py @@ -5,7 +5,8 @@ import tempfile import MySQLdb import binascii -import subprocess +from difflib import unified_diff +import sys import clogging log = logging.getLogger(__name__) @@ -26,6 +27,7 @@ def setup_logging(logger, debug=False): else: logger.setLevel(logging.INFO) + setup_logging(log) @@ -116,6 +118,35 @@ def get_master(connection): return cur.fetchone()['Master_Host'] +# Colorize diff +# http://stackoverflow.com/questions/287871/print-in-terminal-with-colors-using-python +def _green(line): + if sys.stdout.isatty(): + return '\033[92m' + line + '\033[0m' + else: + return line + + +def _red(line): + if sys.stdout.isatty(): + return '\033[91m' + line + '\033[0m' + else: + return line + + +def diff(master_lines, slave_lines): + result = "" + for line in unified_diff(sorted(master_lines), sorted(slave_lines)): + if not line.startswith('---') and not line.startswith('+++'): + if line.startswith('+'): + result += _green(line) + elif line.startswith('-'): + result += _red(line) + else: + result += line + return result + + def get_inconsistencies(db, tbl, slave, user, passwd, ch_db='percona', ch_tbl='checksums'): try: @@ -255,16 +286,11 @@ def get_inconsistencies(db, tbl, slave, user, passwd, os.write(slave_f, "\n") os.close(slave_f) - # Feed diff with the records from the master and slave - # to show the difference to a user - cmd = ["diff", "-u", master_filename, slave_filename] - proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - cout, cerr = proc.communicate() - log.info("Differences between slave %s and its master:\n" - % slave + cout) - if cerr: - log.error(cerr) + diffs = diff(open(master_filename).readlines(), + open(slave_filename).readlines()) + log.info("Differences between slave %s and its master:" % slave) + print(diffs) + os.remove(master_filename) os.remove(slave_filename) except MySQLdb.Error as err: diff --git a/vagrant/Vagrantfile b/vagrant/Vagrantfile index 7852d9d..6fad4f5 100644 --- a/vagrant/Vagrantfile +++ b/vagrant/Vagrantfile @@ -17,11 +17,15 @@ nodes = [ VAGRANTFILE_API_VERSION = '2' $script = <