diff --git a/.travis.yml b/.travis.yml index 58593f4..ff04400 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,7 @@ deploy: provider: pypi user: twindb env: -- TOXENV=flake8 +- TOXENV=lint - TOXENV=cov - TOXENV=py27 - TOXENV=py26 diff --git a/Makefile b/Makefile index 5496d31..8f15a21 100644 --- a/Makefile +++ b/Makefile @@ -48,7 +48,7 @@ clean-test: clean-pyc ## remove test and coverage artifacts rm -fr htmlcov/ lint: ## check style with flake8 - tox -e flake8 + pylint twindb_table_compare vagrant-up: cd vagrant && vagrant up @@ -77,6 +77,8 @@ upgrade-requirements: ## Upgrade requirements test: ## run tests quickly with the default Python py.test -vx tests/unit/ +test-functional: ## run functional tests + py.test -vx tests/functional/ test-all: ## run tests on every Python version with tox tox @@ -92,9 +94,8 @@ docs: ## generate Sphinx HTML documentation, including API docs servedocs: docs ## compile the docs watching for changes watchmedo shell-command -p '*.rst' -c '$(MAKE) -C docs html' -R -D . -release: clean ## package and upload a release - python setup.py sdist upload - python setup.py bdist_wheel upload +coverage: + py.test --cov=twindb_table_compare --cov-report term-missing tests/unit dist: clean ## builds source and wheel package python setup.py sdist diff --git a/pylintrc b/pylintrc new file mode 100644 index 0000000..7470baa --- /dev/null +++ b/pylintrc @@ -0,0 +1,407 @@ +[MASTER] + +# Specify a configuration file. +#rcfile= + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +#init-hook= + +# Add files or directories to the blacklist. They should be base names, not +# paths. +ignore=CVS + +# Add files or directories matching the regex patterns to the blacklist. The +# regex matches against base names, not paths. +ignore-patterns= + +# Pickle collected data for later comparisons. +persistent=yes + +# List of plugins (as comma separated values of python modules names) to load, +# usually to register additional checkers. +load-plugins= + +# Use multiple processes to speed up Pylint. +jobs=1 + +# Allow loading of arbitrary C extensions. Extensions are imported into the +# active Python interpreter and may run arbitrary code. +unsafe-load-any-extension=no + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code +extension-pkg-whitelist= + +# Allow optimization of some AST trees. This will activate a peephole AST +# optimizer, which will apply various small optimizations. For instance, it can +# be used to obtain the result of joining multiple strings with the addition +# operator. Joining a lot of strings can lead to a maximum recursion error in +# Pylint and this flag can prevent that. It has one side effect, the resulting +# AST will be different than the one from reality. This option is deprecated +# and it will be removed in Pylint 2.0. +optimize-ast=no + + +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED +confidence= + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where +# it should appear only once). See also the "--disable" option for examples. +#enable= + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once).You can also use "--disable=all" to +# disable everything first and then reenable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use"--disable=all --enable=classes +# --disable=W" +disable=import-star-module-level,old-octal-literal,oct-method,print-statement,unpacking-in-except,parameter-unpacking,backtick,old-raise-syntax,old-ne-operator,long-suffix,dict-view-method,dict-iter-method,metaclass-assignment,next-method-called,raising-string,indexing-exception,raw_input-builtin,long-builtin,file-builtin,execfile-builtin,coerce-builtin,cmp-builtin,buffer-builtin,basestring-builtin,apply-builtin,filter-builtin-not-iterating,using-cmp-argument,useless-suppression,range-builtin-not-iterating,suppressed-message,no-absolute-import,old-division,cmp-method,reload-builtin,zip-builtin-not-iterating,intern-builtin,unichr-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,input-builtin,round-builtin,hex-method,nonzero-method,map-builtin-not-iterating + + +[REPORTS] + +# Set the output format. Available formats are text, parseable, colorized, msvs +# (visual studio) and html. You can also give a reporter class, eg +# mypackage.mymodule.MyReporterClass. +output-format=text + +# Put messages in a separate file for each module / package specified on the +# command line instead of printing them on stdout. Reports (if any) will be +# written in a file name "pylint_global.[txt|html]". This option is deprecated +# and it will be removed in Pylint 2.0. +files-output=no + +# Tells whether to display a full report or only the messages +reports=yes + +# Python expression which should return a note less than 10 (10 is the highest +# note). You have access to the variables errors warning, statement which +# respectively contain the number of errors / warnings messages and the total +# number of statements analyzed. This is used by the global evaluation report +# (RP0004). +evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details +#msg-template= + + +[BASIC] + +# Good variable names which should always be accepted, separated by a comma +good-names=i,j,k,ex,Run,_ + +# Bad variable names which should always be refused, separated by a comma +bad-names=foo,bar,baz,toto,tutu,tata + +# Colon-delimited sets of names that determine each other's naming style when +# the name regexes allow several styles. +name-group= + +# Include a hint for the correct naming format with invalid-name +include-naming-hint=no + +# List of decorators that produce properties, such as abc.abstractproperty. Add +# to this list to register other decorators that produce valid properties. +property-classes=abc.abstractproperty + +# Regular expression matching correct function names +function-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for function names +function-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct variable names +variable-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for variable names +variable-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct constant names +const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ + +# Naming hint for constant names +const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$ + +# Regular expression matching correct attribute names +attr-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for attribute names +attr-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct argument names +argument-rgx=[a-z_][a-z0-9_]{2,30}$|db$ + +# Naming hint for argument names +argument-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct class attribute names +class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ + +# Naming hint for class attribute names +class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ + +# Regular expression matching correct inline iteration names +inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ + +# Naming hint for inline iteration names +inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$ + +# Regular expression matching correct class names +class-rgx=[A-Z_][a-zA-Z0-9]+$ + +# Naming hint for class names +class-name-hint=[A-Z_][a-zA-Z0-9]+$ + +# Regular expression matching correct module names +module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ + +# Naming hint for module names +module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ + +# Regular expression matching correct method names +method-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for method names +method-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=^_ + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=-1 + + +[ELIF] + +# Maximum number of nested blocks for function / method body +max-nested-blocks=5 + + +[FORMAT] + +# Maximum number of characters on a single line. +max-line-length=80 + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines=^\s*(# )??$ + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=no + +# List of optional constructs for which whitespace checking is disabled. `dict- +# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. +# `trailing-comma` allows a space between comma and closing bracket: (a, ). +# `empty-line` allows space-only lines. +no-space-check=trailing-comma,dict-separator + +# Maximum number of lines in a module +max-module-lines=1000 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format= + + +[LOGGING] + +# Logging modules to check that the string format arguments are in logging +# function parameter format +logging-modules=logging + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=FIXME,XXX,TODO + + +[SIMILARITIES] + +# Minimum lines number of a similarity. +min-similarity-lines=4 + +# Ignore comments when computing similarities. +ignore-comments=yes + +# Ignore docstrings when computing similarities. +ignore-docstrings=yes + +# Ignore imports when computing similarities. +ignore-imports=no + + +[SPELLING] + +# Spelling dictionary name. Available dictionaries: none. To make it working +# install python-enchant package. +spelling-dict= + +# List of comma separated words that should not be checked. +spelling-ignore-words= + +# A path to a file that contains private dictionary; one word per line. +spelling-private-dict-file= + +# Tells whether to store unknown words to indicated private dictionary in +# --spelling-private-dict-file option instead of raising a message. +spelling-store-unknown-words=no + + +[TYPECHECK] + +# Tells whether missing members accessed in mixin class should be ignored. A +# mixin class is detected if its name ends with "mixin" (case insensitive). +ignore-mixin-members=yes + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis. It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules= + +# List of class names for which member attributes should not be checked (useful +# for classes with dynamically set attributes). This supports the use of +# qualified names. +ignored-classes=optparse.Values,thread._local,_thread._local + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members= + +# List of decorators that produce context managers, such as +# contextlib.contextmanager. Add to this list to register other decorators that +# produce valid context managers. +contextmanager-decorators=contextlib.contextmanager + + +[VARIABLES] + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# A regular expression matching the name of dummy variables (i.e. expectedly +# not used). +dummy-variables-rgx=(_+[a-zA-Z0-9]*?$)|dummy + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid to define new builtins when possible. +additional-builtins= + +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_,_cb + +# List of qualified module names which can have objects that can redefine +# builtins. +redefining-builtins-modules=six.moves,future.builtins + + +[CLASSES] + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__,__new__,setUp + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=mcs + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict,_fields,_replace,_source,_make + + +[DESIGN] + +# Maximum number of arguments for function / method +max-args=5 + +# Argument names that match this expression will be ignored. Default to name +# with leading underscore +ignored-argument-names=_.* + +# Maximum number of locals for function / method body +max-locals=15 + +# Maximum number of return / yield for function / method body +max-returns=6 + +# Maximum number of branch for function / method body +max-branches=12 + +# Maximum number of statements in function / method body +max-statements=50 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of attributes for a class (see R0902). +max-attributes=7 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=2 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + +# Maximum number of boolean expressions in a if statement +max-bool-expr=5 + + +[IMPORTS] + +# Deprecated modules which should not be used, separated by a comma +deprecated-modules=regsub,TERMIOS,Bastion,rexec + +# Create a graph of every (i.e. internal and external) dependencies in the +# given file (report RP0402 must not be disabled) +import-graph= + +# Create a graph of external dependencies in the given file (report RP0402 must +# not be disabled) +ext-import-graph= + +# Create a graph of internal dependencies in the given file (report RP0402 must +# not be disabled) +int-import-graph= + +# Force import order to recognize a module as part of the standard +# compatibility libraries. +known-standard-library= + +# Force import order to recognize a module as part of a third party library. +known-third-party=enchant + +# Analyse import fallback blocks. This can be used to support both Python 2 and +# 3 compatible code, which means that the block might have code that exists +# only in one or another interpreter, leading to false positives when analysed. +analyse-fallback-blocks=no + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when being caught. Defaults to +# "Exception" +overgeneral-exceptions=Exception diff --git a/requirements.in b/requirements.in index d727c0c..38f0f52 100644 --- a/requirements.in +++ b/requirements.in @@ -1,2 +1,3 @@ Click>=6.0 mysql +logutils diff --git a/requirements.txt b/requirements.txt index 858c737..f07cadf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,5 +5,6 @@ # pip-compile --no-index --output-file requirements.txt requirements.in # Click==6.7 +logutils==0.3.4.1 MySQL-python==1.2.5 # via mysql mysql==0.0.1 diff --git a/requirements_dev.in b/requirements_dev.in index 5c8d455..faa2d57 100644 --- a/requirements_dev.in +++ b/requirements_dev.in @@ -7,3 +7,4 @@ coverage codecov pytest-cov Sphinx +pylint diff --git a/requirements_dev.txt b/requirements_dev.txt index 0867488..eed1c80 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -5,33 +5,45 @@ # pip-compile --no-index --output-file requirements_dev.txt requirements_dev.in # alabaster==0.7.9 # via sphinx +appdirs==1.4.0 # via setuptools argparse==1.4.0 # via codecov +astroid==1.4.9 # via pylint babel==2.3.4 # via sphinx +backports.functools-lru-cache==1.3 # via pylint bumpversion==0.5.3 codecov==2.0.5 -configparser==3.5.0 # via flake8 -coverage==4.2 +configparser==3.5.0 # via flake8, pylint +coverage==4.3.4 docutils==0.13.1 # via sphinx enum34==1.1.6 # via flake8 -flake8==3.0.4 +flake8==3.3.0 funcsigs==1.0.2 # via mock imagesize==0.7.1 # via sphinx +isort==4.2.5 # via pylint Jinja2==2.9.5 # via sphinx +lazy-object-proxy==1.2.2 # via astroid MarkupSafe==0.23 # via jinja2 -mccabe==0.5.3 # via flake8 +mccabe==0.6.1 # via flake8, pylint mock==2.0.0 +packaging==16.8 # via setuptools pbr==1.10.0 # via mock -pluggy==0.3.1 # via tox +pluggy==0.4.0 # via tox py==1.4.32 # via pytest, tox -pycodestyle==2.0.0 # via flake8 -pyflakes==1.2.3 # via flake8 +pycodestyle==2.3.1 # via flake8 +pyflakes==1.5.0 # via flake8 Pygments==2.2.0 # via sphinx +pylint==1.6.5 +pyparsing==2.1.10 # via packaging pytest-cov==2.4.0 -pytest==3.0.2 +pytest==3.0.6 pytz==2016.10 # via babel -requests==2.13.0 # via codecov -six==1.10.0 # via mock, sphinx +requests==2.13.0 # via codecov, sphinx +six==1.10.0 # via astroid, mock, packaging, pylint, setuptools, sphinx snowballstemmer==1.2.1 # via sphinx -sphinx==1.4.5 -tox==2.3.1 +Sphinx==1.5.2 +tox==2.6.0 virtualenv==15.1.0 # via tox +wrapt==1.10.8 # via astroid + +# The following packages are considered to be unsafe in a requirements file: +# setuptools # via pytest diff --git a/setup.cfg b/setup.cfg index 2616b7c..4afe12b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.0.2 +current_version = 1.1.0 commit = True tag = False diff --git a/setup.py b/setup.py index 5dd1dd1..74ab4be 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ setup( name='twindb_table_compare', - version='1.0.2', + version='1.1.0', description="TwinDB Table Compare reads percona.checksums from the master " "and slave and shows what records are difference " "if there are any inconsistencies.", diff --git a/tests/functional/test_twindb_table_compare_functional.py b/tests/functional/test_twindb_table_compare_functional.py index 55a4f03..1707023 100644 --- a/tests/functional/test_twindb_table_compare_functional.py +++ b/tests/functional/test_twindb_table_compare_functional.py @@ -11,10 +11,7 @@ import MySQLdb import pytest -from click.testing import CliRunner - -from twindb_table_compare import cli -from twindb_table_compare.twindb_table_compare import is_printable, \ +from twindb_table_compare.compare import is_printable, \ get_chunk_index, get_index_fields, get_boundary, get_master @@ -40,16 +37,6 @@ def slave_connection(mysql_cred): passwd=mysql_cred['password']) -def test_command_line_interface(): - runner = CliRunner() - result = runner.invoke(cli.main) - assert result.exit_code == 0 - # assert 'twindb_table_compare.cli.main' in result.output - help_result = runner.invoke(cli.main, ['--help']) - assert help_result.exit_code == 0 - # assert '--help Show this message and exit.' in help_result.output - - @pytest.mark.parametrize('input_str,result', [ ( 'foo', diff --git a/tests/unit/test_twindb_table_compare.py b/tests/unit/test_twindb_table_compare.py index 5b68bc6..1eac759 100644 --- a/tests/unit/test_twindb_table_compare.py +++ b/tests/unit/test_twindb_table_compare.py @@ -8,22 +8,31 @@ Tests for `twindb_table_compare` module. """ import binascii + +import mock import pytest from click.testing import CliRunner -from twindb_table_compare import cli -from twindb_table_compare.twindb_table_compare import is_printable, diff +from twindb_table_compare import cli, __version__ +from twindb_table_compare.compare import is_printable, diff def test_command_line_interface(): runner = CliRunner() - result = runner.invoke(cli.main) - assert result.exit_code == 0 help_result = runner.invoke(cli.main, ['--help']) assert help_result.exit_code == 0 +@mock.patch('twindb_table_compare.cli.get_inconsistencies') +def test_version(mock_get_inconsistencies): + runner = CliRunner() + mock_get_inconsistencies.side_effect = Exception + help_result = runner.invoke(cli.main, ['--version']) + assert help_result.output.strip('\n') == __version__ + assert help_result.exit_code == 0 + + @pytest.mark.parametrize('input_str,result', [ ( 'foo', @@ -66,8 +75,567 @@ def test_is_printable(input_str, result): '3882\t2016-04-20 14:57:31\n', '3882\t2016-04-20 14:57:31\n' ], - "" + """@@ -1,3 +1,3 @@ ++3937\t2016-05-13 14:32:53 + 3882\t2016-04-20 14:57:31 + 3882\t2016-04-20 14:57:31 +-3937\t2016-05-13 14:32:53 +""" + ), + ( + [ + "*************************** 1. row ***************************\n", + " Host: localhost\n", + " User: root\n", + " Password: \n", + " Select_priv: Y\n", + " Insert_priv: Y\n", + " Update_priv: Y\n", + " Delete_priv: Y\n", + " Create_priv: Y\n", + " Drop_priv: Y\n", + " Reload_priv: Y\n", + " Shutdown_priv: Y\n", + " Process_priv: Y\n", + " File_priv: Y\n", + " Grant_priv: Y\n", + " References_priv: Y\n", + " Index_priv: Y\n", + " Alter_priv: Y\n", + " Show_db_priv: Y\n", + " Super_priv: Y\n", + " Create_tmp_table_priv: Y\n", + " Lock_tables_priv: Y\n", + " Execute_priv: Y\n", + " Repl_slave_priv: Y\n", + " Repl_client_priv: Y\n", + " Create_view_priv: Y\n", + " Show_view_priv: Y\n", + " Create_routine_priv: Y\n", + " Alter_routine_priv: Y\n", + " Create_user_priv: Y\n", + " Event_priv: Y\n", + " Trigger_priv: Y\n", + "Create_tablespace_priv: Y\n", + " ssl_type: \n", + " HEX(ssl_cipher): \n", + " HEX(x509_issuer): \n", + " HEX(x509_subject): \n", + " max_questions: 0\n", + " max_updates: 0\n", + " max_connections: 0\n", + " max_user_connections: 0\n", + " plugin: mysql_native_password\n", + " authentication_string: \n", + " password_expired: N\n", + "*************************** 2. row ***************************\n", + " Host: master.box\n", + " User: root\n", + " Password: \n", + " Select_priv: Y\n", + " Insert_priv: Y\n", + " Update_priv: Y\n", + " Delete_priv: Y\n", + " Create_priv: Y\n", + " Drop_priv: Y\n", + " Reload_priv: Y\n", + " Shutdown_priv: Y\n", + " Process_priv: Y\n", + " File_priv: Y\n", + " Grant_priv: Y\n", + " References_priv: Y\n", + " Index_priv: Y\n", + " Alter_priv: Y\n", + " Show_db_priv: Y\n", + " Super_priv: Y\n", + " Create_tmp_table_priv: Y\n", + " Lock_tables_priv: Y\n", + " Execute_priv: Y\n", + " Repl_slave_priv: Y\n", + " Repl_client_priv: Y\n", + " Create_view_priv: Y\n", + " Show_view_priv: Y\n", + " Create_routine_priv: Y\n", + " Alter_routine_priv: Y\n", + " Create_user_priv: Y\n", + " Event_priv: Y\n", + " Trigger_priv: Y\n", + "Create_tablespace_priv: Y\n", + " ssl_type: \n", + " HEX(ssl_cipher): \n", + " HEX(x509_issuer): \n", + " HEX(x509_subject): \n", + " max_questions: 0\n", + " max_updates: 0\n", + " max_connections: 0\n", + " max_user_connections: 0\n", + " plugin: mysql_native_password\n", + " authentication_string: \n", + " password_expired: N\n", + "*************************** 3. row ***************************\n", + " Host: 127.0.0.1\n", + " User: root\n", + " Password: \n", + " Select_priv: Y\n", + " Insert_priv: Y\n", + " Update_priv: Y\n", + " Delete_priv: Y\n", + " Create_priv: Y\n", + " Drop_priv: Y\n", + " Reload_priv: Y\n", + " Shutdown_priv: Y\n", + " Process_priv: Y\n", + " File_priv: Y\n", + " Grant_priv: Y\n", + " References_priv: Y\n", + " Index_priv: Y\n", + " Alter_priv: Y\n", + " Show_db_priv: Y\n", + " Super_priv: Y\n", + " Create_tmp_table_priv: Y\n", + " Lock_tables_priv: Y\n", + " Execute_priv: Y\n", + " Repl_slave_priv: Y\n", + " Repl_client_priv: Y\n", + " Create_view_priv: Y\n", + " Show_view_priv: Y\n", + " Create_routine_priv: Y\n", + " Alter_routine_priv: Y\n", + " Create_user_priv: Y\n", + " Event_priv: Y\n", + " Trigger_priv: Y\n", + "Create_tablespace_priv: Y\n", + " ssl_type: \n", + " HEX(ssl_cipher): \n", + " HEX(x509_issuer): \n", + " HEX(x509_subject): \n", + " max_questions: 0\n", + " max_updates: 0\n", + " max_connections: 0\n", + " max_user_connections: 0\n", + " plugin: mysql_native_password\n", + " authentication_string: \n", + " password_expired: N\n", + "*************************** 4. row ***************************\n", + " Host: ::1\n", + " User: root\n", + " Password: \n", + " Select_priv: Y\n", + " Insert_priv: Y\n", + " Update_priv: Y\n", + " Delete_priv: Y\n", + " Create_priv: Y\n", + " Drop_priv: Y\n", + " Reload_priv: Y\n", + " Shutdown_priv: Y\n", + " Process_priv: Y\n", + " File_priv: Y\n", + " Grant_priv: Y\n", + " References_priv: Y\n", + " Index_priv: Y\n", + " Alter_priv: Y\n", + " Show_db_priv: Y\n", + " Super_priv: Y\n", + " Create_tmp_table_priv: Y\n", + " Lock_tables_priv: Y\n", + " Execute_priv: Y\n", + " Repl_slave_priv: Y\n", + " Repl_client_priv: Y\n", + " Create_view_priv: Y\n", + " Show_view_priv: Y\n", + " Create_routine_priv: Y\n", + " Alter_routine_priv: Y\n", + " Create_user_priv: Y\n", + " Event_priv: Y\n", + " Trigger_priv: Y\n", + "Create_tablespace_priv: Y\n", + " ssl_type: \n", + " HEX(ssl_cipher): \n", + " HEX(x509_issuer): \n", + " HEX(x509_subject): \n", + " max_questions: 0\n", + " max_updates: 0\n", + " max_connections: 0\n", + " max_user_connections: 0\n", + " plugin: mysql_native_password\n", + " authentication_string: \n", + " password_expired: N\n", + "*************************** 5. row ***************************\n", + " Host: localhost\n", + " User: \n", + " Password: \n", + " Select_priv: N\n", + " Insert_priv: N\n", + " Update_priv: N\n", + " Delete_priv: N\n", + " Create_priv: N\n", + " Drop_priv: N\n", + " Reload_priv: N\n", + " Shutdown_priv: N\n", + " Process_priv: N\n", + " File_priv: N\n", + " Grant_priv: N\n", + " References_priv: N\n", + " Index_priv: N\n", + " Alter_priv: N\n", + " Show_db_priv: N\n", + " Super_priv: N\n", + " Create_tmp_table_priv: N\n", + " Lock_tables_priv: N\n", + " Execute_priv: N\n", + " Repl_slave_priv: N\n", + " Repl_client_priv: N\n", + " Create_view_priv: N\n", + " Show_view_priv: N\n", + " Create_routine_priv: N\n", + " Alter_routine_priv: N\n", + " Create_user_priv: N\n", + " Event_priv: N\n", + " Trigger_priv: N\n", + "Create_tablespace_priv: N\n", + " ssl_type: \n", + " HEX(ssl_cipher): \n", + " HEX(x509_issuer): \n", + " HEX(x509_subject): \n", + " max_questions: 0\n", + " max_updates: 0\n", + " max_connections: 0\n", + " max_user_connections: 0\n", + " plugin: mysql_native_password\n", + " authentication_string: NULL\n", + " password_expired: N\n", + "*************************** 6. row ***************************\n", + " Host: master.box\n", + " User: \n", + " Password: \n", + " Select_priv: N\n", + " Insert_priv: N\n", + " Update_priv: N\n", + " Delete_priv: N\n", + " Create_priv: N\n", + " Drop_priv: N\n", + " Reload_priv: N\n", + " Shutdown_priv: N\n", + " Process_priv: N\n", + " File_priv: N\n", + " Grant_priv: N\n", + " References_priv: N\n", + " Index_priv: N\n", + " Alter_priv: N\n", + " Show_db_priv: N\n", + " Super_priv: N\n", + " Create_tmp_table_priv: N\n", + " Lock_tables_priv: N\n", + " Execute_priv: N\n", + " Repl_slave_priv: N\n", + " Repl_client_priv: N\n", + " Create_view_priv: N\n", + " Show_view_priv: N\n", + " Create_routine_priv: N\n", + " Alter_routine_priv: N\n", + " Create_user_priv: N\n", + " Event_priv: N\n", + " Trigger_priv: N\n", + "Create_tablespace_priv: N\n", + " ssl_type: \n", + " HEX(ssl_cipher): \n", + " HEX(x509_issuer): \n", + " HEX(x509_subject): \n", + " max_questions: 0\n", + " max_updates: 0\n", + " max_connections: 0\n", + " max_user_connections: 0\n", + " plugin: mysql_native_password\n", + " authentication_string: NULL\n", + " password_expired: N\n", + ], + [ + "*************************** 1. row ***************************\n", + " Host: localhost\n", + " User: root\n", + " Password: \n", + " Select_priv: Y\n", + " Insert_priv: Y\n", + " Update_priv: Y\n", + " Delete_priv: Y\n", + " Create_priv: Y\n", + " Drop_priv: Y\n", + " Reload_priv: Y\n", + " Shutdown_priv: Y\n", + " Process_priv: Y\n", + " File_priv: Y\n", + " Grant_priv: Y\n", + " References_priv: Y\n", + " Index_priv: Y\n", + " Alter_priv: Y\n", + " Show_db_priv: Y\n", + " Super_priv: Y\n", + " Create_tmp_table_priv: Y\n", + " Lock_tables_priv: Y\n", + " Execute_priv: Y\n", + " Repl_slave_priv: Y\n", + " Repl_client_priv: Y\n", + " Create_view_priv: Y\n", + " Show_view_priv: Y\n", + " Create_routine_priv: Y\n", + " Alter_routine_priv: Y\n", + " Create_user_priv: Y\n", + " Event_priv: Y\n", + " Trigger_priv: Y\n", + "Create_tablespace_priv: Y\n", + " ssl_type: \n", + " HEX(ssl_cipher): \n", + " HEX(x509_issuer): \n", + " HEX(x509_subject): \n", + " max_questions: 0\n", + " max_updates: 0\n", + " max_connections: 0\n", + " max_user_connections: 0\n", + " plugin: mysql_native_password\n", + " authentication_string: \n", + " password_expired: N\n", + "*************************** 2. row ***************************\n", + " Host: slave.box\n", + " User: root\n", + " Password: \n", + " Select_priv: Y\n", + " Insert_priv: Y\n", + " Update_priv: Y\n", + " Delete_priv: Y\n", + " Create_priv: Y\n", + " Drop_priv: Y\n", + " Reload_priv: Y\n", + " Shutdown_priv: Y\n", + " Process_priv: Y\n", + " File_priv: Y\n", + " Grant_priv: Y\n", + " References_priv: Y\n", + " Index_priv: Y\n", + " Alter_priv: Y\n", + " Show_db_priv: Y\n", + " Super_priv: Y\n", + " Create_tmp_table_priv: Y\n", + " Lock_tables_priv: Y\n", + " Execute_priv: Y\n", + " Repl_slave_priv: Y\n", + " Repl_client_priv: Y\n", + " Create_view_priv: Y\n", + " Show_view_priv: Y\n", + " Create_routine_priv: Y\n", + " Alter_routine_priv: Y\n", + " Create_user_priv: Y\n", + " Event_priv: Y\n", + " Trigger_priv: Y\n", + "Create_tablespace_priv: Y\n", + " ssl_type: \n", + " HEX(ssl_cipher): \n", + " HEX(x509_issuer): \n", + " HEX(x509_subject): \n", + " max_questions: 0\n", + " max_updates: 0\n", + " max_connections: 0\n", + " max_user_connections: 0\n", + " plugin: mysql_native_password\n", + " authentication_string: \n", + " password_expired: N\n", + "*************************** 3. row ***************************\n", + " Host: 127.0.0.1\n", + " User: root\n", + " Password: \n", + " Select_priv: Y\n", + " Insert_priv: Y\n", + " Update_priv: Y\n", + " Delete_priv: Y\n", + " Create_priv: Y\n", + " Drop_priv: Y\n", + " Reload_priv: Y\n", + " Shutdown_priv: Y\n", + " Process_priv: Y\n", + " File_priv: Y\n", + " Grant_priv: Y\n", + " References_priv: Y\n", + " Index_priv: Y\n", + " Alter_priv: Y\n", + " Show_db_priv: Y\n", + " Super_priv: Y\n", + " Create_tmp_table_priv: Y\n", + " Lock_tables_priv: Y\n", + " Execute_priv: Y\n", + " Repl_slave_priv: Y\n", + " Repl_client_priv: Y\n", + " Create_view_priv: Y\n", + " Show_view_priv: Y\n", + " Create_routine_priv: Y\n", + " Alter_routine_priv: Y\n", + " Create_user_priv: Y\n", + " Event_priv: Y\n", + " Trigger_priv: Y\n", + "Create_tablespace_priv: Y\n", + " ssl_type: \n", + " HEX(ssl_cipher): \n", + " HEX(x509_issuer): \n", + " HEX(x509_subject): \n", + " max_questions: 0\n", + " max_updates: 0\n", + " max_connections: 0\n", + " max_user_connections: 0\n", + " plugin: mysql_native_password\n", + " authentication_string: \n", + " password_expired: N\n", + "*************************** 4. row ***************************\n", + " Host: ::1\n", + " User: root\n", + " Password: \n", + " Select_priv: Y\n", + " Insert_priv: Y\n", + " Update_priv: Y\n", + " Delete_priv: Y\n", + " Create_priv: Y\n", + " Drop_priv: Y\n", + " Reload_priv: Y\n", + " Shutdown_priv: Y\n", + " Process_priv: Y\n", + " File_priv: Y\n", + " Grant_priv: Y\n", + " References_priv: Y\n", + " Index_priv: Y\n", + " Alter_priv: Y\n", + " Show_db_priv: Y\n", + " Super_priv: Y\n", + " Create_tmp_table_priv: Y\n", + " Lock_tables_priv: Y\n", + " Execute_priv: Y\n", + " Repl_slave_priv: Y\n", + " Repl_client_priv: Y\n", + " Create_view_priv: Y\n", + " Show_view_priv: Y\n", + " Create_routine_priv: Y\n", + " Alter_routine_priv: Y\n", + " Create_user_priv: Y\n", + " Event_priv: Y\n", + " Trigger_priv: Y\n", + "Create_tablespace_priv: Y\n", + " ssl_type: \n", + " HEX(ssl_cipher): \n", + " HEX(x509_issuer): \n", + " HEX(x509_subject): \n", + " max_questions: 0\n", + " max_updates: 0\n", + " max_connections: 0\n", + " max_user_connections: 0\n", + " plugin: mysql_native_password\n", + " authentication_string: \n", + " password_expired: N\n", + "*************************** 5. row ***************************\n", + " Host: localhost\n", + " User: \n", + " Password: \n", + " Select_priv: N\n", + " Insert_priv: N\n", + " Update_priv: N\n", + " Delete_priv: N\n", + " Create_priv: N\n", + " Drop_priv: N\n", + " Reload_priv: N\n", + " Shutdown_priv: N\n", + " Process_priv: N\n", + " File_priv: N\n", + " Grant_priv: N\n", + " References_priv: N\n", + " Index_priv: N\n", + " Alter_priv: N\n", + " Show_db_priv: N\n", + " Super_priv: N\n", + " Create_tmp_table_priv: N\n", + " Lock_tables_priv: N\n", + " Execute_priv: N\n", + " Repl_slave_priv: N\n", + " Repl_client_priv: N\n", + " Create_view_priv: N\n", + " Show_view_priv: N\n", + " Create_routine_priv: N\n", + " Alter_routine_priv: N\n", + " Create_user_priv: N\n", + " Event_priv: N\n", + " Trigger_priv: N\n", + "Create_tablespace_priv: N\n", + " ssl_type: \n", + " HEX(ssl_cipher): \n", + " HEX(x509_issuer): \n", + " HEX(x509_subject): \n", + " max_questions: 0\n", + " max_updates: 0\n", + " max_connections: 0\n", + " max_user_connections: 0\n", + " plugin: mysql_native_password\n", + " authentication_string: NULL\n", + " password_expired: N\n", + "*************************** 6. row ***************************\n", + " Host: slave.box\n", + " User: \n", + " Password: \n", + " Select_priv: N\n", + " Insert_priv: N\n", + " Update_priv: N\n", + " Delete_priv: N\n", + " Create_priv: N\n", + " Drop_priv: N\n", + " Reload_priv: N\n", + " Shutdown_priv: N\n", + " Process_priv: N\n", + " File_priv: N\n", + " Grant_priv: N\n", + " References_priv: N\n", + " Index_priv: N\n", + " Alter_priv: N\n", + " Show_db_priv: N\n", + " Super_priv: N\n", + " Create_tmp_table_priv: N\n", + " Lock_tables_priv: N\n", + " Execute_priv: N\n", + " Repl_slave_priv: N\n", + " Repl_client_priv: N\n", + " Create_view_priv: N\n", + " Show_view_priv: N\n", + " Create_routine_priv: N\n", + " Alter_routine_priv: N\n", + " Create_user_priv: N\n", + " Event_priv: N\n", + " Trigger_priv: N\n", + "Create_tablespace_priv: N\n", + " ssl_type: \n", + " HEX(ssl_cipher): \n", + " HEX(x509_issuer): \n", + " HEX(x509_subject): \n", + " max_questions: 0\n", + " max_updates: 0\n", + " max_connections: 0\n", + " max_user_connections: 0\n", + " plugin: mysql_native_password\n", + " authentication_string: NULL\n", + " password_expired: N\n", + ], + "@@ -43,7 +43,7 @@\n" + " authentication_string: \n" + " password_expired: N\n" + " *************************** 2. row ***************************\n" + "- Host: master.box\n" + "+ Host: slave.box\n" + " User: root\n" + " Password: \n" + " Select_priv: Y\n" + "@@ -219,7 +219,7 @@\n" + " authentication_string: NULL\n" + " password_expired: N\n" + " *************************** 6. row ***************************\n" + "- Host: master.box\n" + "+ Host: slave.box\n" + " User: \n" + " Password: \n" + " Select_priv: N\n" ) ]) def test_diff(master_lines, slave_lines, difference): - assert diff(master_lines, slave_lines) == difference + + actual_diff = diff(master_lines, slave_lines) + assert actual_diff == difference diff --git a/tox.ini b/tox.ini index 4544f75..5c4e28a 100644 --- a/tox.ini +++ b/tox.ini @@ -1,10 +1,9 @@ [tox] -envlist = py26, py27, flake8, cov +envlist = py26, py27, lint, cov -[testenv:flake8] +[testenv:lint] basepython=python -deps=flake8 -commands=flake8 twindb_table_compare +commands=pylint twindb_table_compare [testenv:cov] deps=-rrequirements_dev.txt diff --git a/twindb_table_compare/__init__.py b/twindb_table_compare/__init__.py index b96ec20..c437e15 100644 --- a/twindb_table_compare/__init__.py +++ b/twindb_table_compare/__init__.py @@ -1,5 +1,43 @@ # -*- coding: utf-8 -*- +""" +Module to read pr-table-checksum's result table (percona.checksums) +and show user which records are actually different. +""" +import logging +from logutils.colorize import ColorizingStreamHandler __author__ = 'Aleksandr Kuzminsky' __email__ = 'aleks@twindb.com' -__version__ = '1.0.2' +__version__ = '1.1.0' + +LOG = logging.getLogger(__name__) + + +def setup_logging(logger, debug=False, color=True): + """ + Configure logging. + + :param logger: Logger to configure. + :type logger: Logger + :param debug: If True - print debug messages + :param color: If True - print colored messages + """ + + fmt_str = "%(asctime)s: %(levelname)s:" \ + " %(module)s.%(funcName)s():%(lineno)d: %(message)s" + + logger.handlers = [] + if color: + colored_console_handler = ColorizingStreamHandler() + colored_console_handler.level_map[logging.INFO] = (None, 'cyan', False) + colored_console_handler .setFormatter(logging.Formatter(fmt_str)) + logger.addHandler(colored_console_handler) + else: + console_handler = logging.StreamHandler() + console_handler.setFormatter(logging.Formatter(fmt_str)) + logger.addHandler(console_handler) + + if debug: + logger.setLevel(logging.DEBUG) + else: + logger.setLevel(logging.INFO) diff --git a/twindb_table_compare/cli.py b/twindb_table_compare/cli.py index 07b2d71..9fd1ace 100644 --- a/twindb_table_compare/cli.py +++ b/twindb_table_compare/cli.py @@ -1,9 +1,18 @@ # -*- coding: utf-8 -*- +""" +Command line routines +""" +from __future__ import print_function +import pwd import os +import MySQLdb import click -import pwd -import twindb_table_compare + +from twindb_table_compare.compare import get_inconsistencies, \ + get_inconsistent_tables + +from . import __version__, setup_logging, LOG @click.command() @@ -18,21 +27,33 @@ @click.option('--vertical', default=False, is_flag=True, help='Print result vertically. ' 'Otherwise will print one record in one line') -@click.argument('slave', default='localhost', required=False) -def main(user, password, db, tbl, slave, vertical): +@click.option('--version', is_flag=True, + help='Print version and exit', default=False) +@click.option('--debug', is_flag=True, + help='Print debug messages', default=False) +@click.option('--color/--no-color', is_flag=True, + help='Print colored log messages', default=True) +@click.argument('slave', + default='localhost', + required=False) # pylint: disable=too-many-arguments +def main(user, password, db, tbl, slave, + vertical, debug, version, color): """twindb_table_compare reads percona.checksums from the master and slave and shows records that differ if there are any inconsistencies.""" + if version: + print(__version__) + exit(0) - for d, t in twindb_table_compare.get_inconsistent_tables(slave, - user, - password, - ch_db=db, - ch_tbl=tbl): - twindb_table_compare.get_inconsistencies(d, t, slave, user, - password, - ch_db=db, ch_tbl=tbl, - vertical=vertical) - - -if __name__ == "__main__": - main() + setup_logging(LOG, debug=debug, color=color) + try: + for database, table in get_inconsistent_tables(slave, + user, + password, + ch_db=db, + ch_tbl=tbl): + get_inconsistencies(database, table, slave, user, password, + ch_db=db, ch_tbl=tbl, + vertical=vertical, color=color) + except MySQLdb.Error as err: # pylint: disable=no-member + LOG.error(err) + exit(1) diff --git a/twindb_table_compare/clogging.py b/twindb_table_compare/clogging.py deleted file mode 100644 index 1b2b2d4..0000000 --- a/twindb_table_compare/clogging.py +++ /dev/null @@ -1,155 +0,0 @@ -# -# Copyright (C) 2010-2012 Vinay Sajip. -# All rights reserved. Licensed under the new BSD license. -# -import ctypes -import logging -import logging.handlers -import os -import sys - - -class ColorizingStreamHandler(logging.StreamHandler): - def __init__(self): - logging.StreamHandler.__init__(self, sys.stdout) - - # color names to indices - color_map = { - 'black': 0, - 'red': 1, - 'green': 2, - 'yellow': 3, - 'blue': 4, - 'magenta': 5, - 'cyan': 6, - 'white': 7, - } - - # levels to (background, foreground, bold/intense) - if os.name == 'nt': - level_map = { - logging.DEBUG: (None, 'blue', True), - logging.INFO: (None, 'white', False), - logging.WARNING: (None, 'yellow', True), - logging.ERROR: (None, 'red', True), - logging.CRITICAL: ('red', 'white', True), - } - else: - level_map = { - logging.DEBUG: (None, 'blue', False), - logging.INFO: (None, 'green', False), - logging.WARNING: (None, 'yellow', False), - logging.ERROR: (None, 'red', False), - logging.CRITICAL: ('red', 'white', True), - } - csi = '\x1b[' - reset = '\x1b[0m' - - @property - def is_tty(self): - isatty = getattr(self.stream, 'isatty', None) - return isatty and isatty() - - def emit(self, record): - try: - message = self.format(record) - stream = self.stream - if not self.is_tty: - stream.write(message) - else: - self.output_colorized(message) - stream.write(getattr(self, 'terminator', '\n')) - self.flush() - except (KeyboardInterrupt, SystemExit): - raise - # except: - # self.handleError(record) - - if os.name != 'nt': - def output_colorized(self, message): - self.stream.write(message) - else: - import re - ansi_esc = re.compile(r'\x1b\[((?:\d+)(?:;(?:\d+))*)m') - - nt_color_map = { - 0: 0x00, # black - 1: 0x04, # red - 2: 0x02, # green - 3: 0x06, # yellow - 4: 0x01, # blue - 5: 0x05, # magenta - 6: 0x03, # cyan - 7: 0x07, # white - } - - def output_colorized(self, message): - parts = self.ansi_esc.split(message) - write = self.stream.write - h = None - fd = getattr(self.stream, 'fileno', None) - if fd is not None: - fd = fd() - if fd in (1, 2): # stdout or stderr - h = ctypes.windll.kernel32.GetStdHandle(-10 - fd) - while parts: - text = parts.pop(0) - if text: - write(text) - if parts: - params = parts.pop(0) - if h is not None: - params = [int(p) for p in params.split(';')] - color = 0 - for p in params: - if 40 <= p <= 47: - color |= self.nt_color_map[p - 40] << 4 - elif 30 <= p <= 37: - color |= self.nt_color_map[p - 30] - elif p == 1: - color |= 0x08 # foreground intensity on - elif p == 0: # reset to default color - color = 0x07 - else: - pass # error condition ignored - ctypes.windll.kernel32.SetConsoleTextAttribute(h, - color) - - def colorize(self, message, record): - if record.levelno in self.level_map: - bg, fg, bold = self.level_map[record.levelno] - params = [] - if bg in self.color_map: - params.append(str(self.color_map[bg] + 40)) - if fg in self.color_map: - params.append(str(self.color_map[fg] + 30)) - if bold: - params.append('1') - if params: - message = ''.join((self.csi, ';'.join(params), - 'm', message, self.reset)) - return message - - def format(self, record): - message = logging.StreamHandler.format(self, record) - if self.is_tty: - # Don't colorize any traceback - parts = message.split('\n', 1) - parts[0] = self.colorize(parts[0], record) - message = '\n'.join(parts) - return message - - -def main(): - root = logging.getLogger() - root.setLevel(logging.DEBUG) - root.addHandler(ColorizingStreamHandler()) - logging.debug('DEBUG') - logging.info('INFO') - logging.warning('WARNING') - logging.error('ERROR') - logging.critical('CRITICAL') - - -if __name__ == '__main__': - main() diff --git a/twindb_table_compare/twindb_table_compare.py b/twindb_table_compare/compare.py similarity index 50% rename from twindb_table_compare/twindb_table_compare.py rename to twindb_table_compare/compare.py index 4981a54..fb20bde 100644 --- a/twindb_table_compare/twindb_table_compare.py +++ b/twindb_table_compare/compare.py @@ -1,37 +1,20 @@ # -*- coding: utf-8 -*- -import logging -import os -import string -import tempfile -import MySQLdb +""" +Functions to find and print differences +""" +from __future__ import print_function import binascii from difflib import unified_diff -import sys - +import os +import string from subprocess import Popen, PIPE +import sys +import tempfile -import clogging - -log = logging.getLogger(__name__) - - -def setup_logging(logger, debug=False): - - fmt_str = "%(asctime)s: %(levelname)s:" \ - " %(module)s.%(funcName)s():%(lineno)d: %(message)s" - - # console_handler = logging.StreamHandler() - console_handler = clogging.ColorizingStreamHandler() - console_handler.setFormatter(logging.Formatter(fmt_str)) - logger.handlers = [] - logger.addHandler(console_handler) - if debug: - logger.setLevel(logging.DEBUG) - else: - logger.setLevel(logging.INFO) +import MySQLdb -setup_logging(log) +from . import LOG def is_printable(str_value): @@ -43,7 +26,8 @@ def is_printable(str_value): return set(str_value).issubset(string.printable) -def get_chunk_index(connection, db, tbl, chunk, +def get_chunk_index(connection, db, # pylint: disable=too-many-arguments + tbl, chunk, ch_db='percona', ch_tbl='checksums'): """ Get index that was used to cut the chunk @@ -52,13 +36,15 @@ def get_chunk_index(connection, db, tbl, chunk, :param db: database of the chunk :param tbl: table of the chunk :param chunk: chunk id + :param ch_db: Database where checksums are stored. Default percona. + :param ch_tbl: Table where checksums are stored. Default checksums. :return: index name or None if no index was used """ cur = connection.cursor() query = "SELECT chunk_index FROM `%s`.`%s` " \ "WHERE db='%s' AND tbl='%s' AND chunk = %s" - log.info('Executing %s' % query % (ch_db, ch_tbl, db, tbl, chunk)) + LOG.info('Executing %s', query % (ch_db, ch_tbl, db, tbl, chunk)) cur.execute(query % (ch_db, ch_tbl, db, tbl, chunk)) return cur.fetchone()[0] @@ -79,7 +65,7 @@ def get_index_fields(connection, db, tbl, index): "AND TABLE_NAME='%s' " \ "AND INDEX_NAME='%s' " \ "ORDER BY SEQ_IN_INDEX" - log.info('Executing %s' % query % (db, tbl, index)) + LOG.info('Executing %s', query % (db, tbl, index)) cur.execute(query % (db, tbl, index)) cols = [] for row in cur.fetchall(): @@ -87,7 +73,8 @@ def get_index_fields(connection, db, tbl, index): return cols -def get_boundary(connection, db, tbl, chunk, +def get_boundary(connection, # pylint: disable=too-many-arguments + db, tbl, chunk, ch_db='percona', ch_tbl='checksums'): """ Get lower and upper boundary values of a chunk @@ -102,7 +89,7 @@ def get_boundary(connection, db, tbl, chunk, cur = connection.cursor() query = "SELECT lower_boundary, upper_boundary FROM `%s`.`%s` " \ "WHERE db='%s' AND tbl='%s' AND chunk = %s" - log.info('Executing %s' % query % (ch_db, ch_tbl, db, tbl, chunk)) + LOG.info('Executing %s', query % (ch_db, ch_tbl, db, tbl, chunk)) cur.execute(query % (ch_db, ch_tbl, db, tbl, chunk)) return cur.fetchone() @@ -116,13 +103,13 @@ def get_master(connection): """ cur = connection.cursor(MySQLdb.cursors.DictCursor) query = "SHOW SLAVE STATUS" - log.info('Executing %s' % query) + LOG.info('Executing %s', query) cur.execute(query) return cur.fetchone()['Master_Host'] # Colorize diff -# http://stackoverflow.com/questions/287871/print-in-terminal-with-colors-using-python +# https://goo.gl/GqSyoj def _green(line): if sys.stdout.isatty(): return '\033[92m' + line + '\033[0m' @@ -137,18 +124,39 @@ def _red(line): return line -def diff(master_lines, slave_lines): +def diff(master_lines, slave_lines, color=True): + """ + Find differences between two set of lines. + + :param master_lines: First set of lines + :type master_lines: list + :param slave_lines: Second set of lines + :type slave_lines: list + :param color: If True return colored diff + :type color: bool + :return: Difference between two set of lines + :rtype: str + """ result = "" - for line in unified_diff(sorted(master_lines), sorted(slave_lines)): + for line in unified_diff(master_lines, slave_lines): if not line.startswith('---') and not line.startswith('+++'): if not line.endswith('\n'): # print(result) line += '\n' - pass + if line.startswith('+'): - result += _green(line) + + if color: + result += _green(line) + else: + result += line + elif line.startswith('-'): - result += _red(line) + + if color: + result += _red(line) + else: + result += line else: result += line @@ -156,6 +164,19 @@ def diff(master_lines, slave_lines): def get_fileds(conn, db, tbl): + """ + Construct fields list string for a SELECT. + If a field is a binary type (BLOB, VARBINARY) then HEX() it. + + :param conn: MySQL connection. + :type conn: Connection + :param db: Database name. + :type db: str + :param tbl: Table name. + :type tbl: str + :return: A comma separated list of fields. + :rtype: str + """ query = "SELECT COLUMN_NAME, DATA_TYPE " \ "FROM information_schema.COLUMNS " \ "WHERE TABLE_SCHEMA='{db}' AND TABLE_NAME='{tbl}' " \ @@ -175,13 +196,21 @@ def get_fileds(conn, db, tbl): return ', '.join(fields) -def build_chunk_query(db, tbl, chunk, conn, ch_db='percona', +# pylint: disable=too-many-arguments,too-many-locals,too-many-branches,too-many-statements +def build_chunk_query(db, + tbl, + chunk, + conn, + ch_db='percona', ch_tbl='checksusm'): + """For a given database, table and chunk number construct + a SELECT query that would return records in this chunk. + """ - log.info("# %s.%s, chunk %d" % (db, tbl, chunk)) + LOG.info("# %s.%s, chunk %d", db, tbl, chunk) chunk_index = get_chunk_index(conn, db, tbl, chunk, ch_db=ch_db, ch_tbl=ch_tbl) - log.info("# chunk index: %s" % chunk_index) + LOG.info("# chunk index: %s", chunk_index) where = "WHERE" if chunk_index: index_fields = get_index_fields(conn, @@ -199,27 +228,27 @@ def build_chunk_query(db, tbl, chunk, conn, ch_db='percona', clause_fields = [] v_num = 0 where += " (0 " - op = ">" + oper = ">" for index_field in index_fields: clause_fields.append(index_field) where += " OR ( 1" for clause_field in clause_fields: if clause_field == clause_fields[len(clause_fields) - 1]: if clause_field == index_field_last: - op = ">=" + oper = ">=" else: - op = ">" - v = lower_boundaries[v_num] + oper = ">" + value = lower_boundaries[v_num] v_num += 1 - if is_printable(v): + if is_printable(value): where += (" AND `%s` %s '%s'" - % (clause_field, op, v)) + % (clause_field, oper, value)) else: - v = ("UNHEX('%s')" - % binascii.hexlify(str(v))) + value = ("UNHEX('%s')" + % binascii.hexlify(str(value))) where += (" AND `%s` %s %s" - % (clause_field, op, v)) - op = "=" + % (clause_field, oper, value)) + oper = "=" where += " )" where += " )" @@ -227,41 +256,56 @@ def build_chunk_query(db, tbl, chunk, conn, ch_db='percona', clause_fields = [] v_num = 0 where += " AND ( 0" - op = "<" + oper = "<" for index_field in index_fields: clause_fields.append(index_field) where += " OR ( 1" for clause_field in clause_fields: if clause_field == clause_fields[len(clause_fields) - 1]: if clause_field == index_field_last: - op = "<=" + oper = "<=" else: - op = "<" - v = upper_boundaries[v_num] + oper = "<" + value = upper_boundaries[v_num] v_num += 1 - if is_printable(v): + if is_printable(value): where += (" AND `%s` %s '%s'" - % (clause_field, op, v)) + % (clause_field, oper, value)) else: - v = ("UNHEX('%s')" - % binascii.hexlify(str(v))) + value = ("UNHEX('%s')" + % binascii.hexlify(str(value))) where += (" AND `%s` %s %s" - % (clause_field, op, v)) - op = "=" + % (clause_field, oper, value)) + oper = "=" where += " )" where += " )" else: where += " 1" fields = get_fileds(conn, db, tbl) - query = "SELECT %s FROM `%s`.`%s` %s" % (fields, db, tbl, where) + query = "SELECT %s FROM `%s`.`%s` USING (PRIMARY) %s" \ + % (fields, db, tbl, where) return query -def print_horizontal(cur_master, cur_slave, query): +def print_horizontal(cur_master, cur_slave, query, color=True): + """ + Find and return differences in horizontal format i.e. one line + - one record + + :param cur_master: MySQLdb cursor on master + :type cur_master: Cursor + :param cur_slave: MySQLdb cursor on slave + :type cur_slave: Cursor + :param query: Query to find records in a chunk we compare + :type query: str + :param color: If True - produce colorful output + :return: Differences in a chunk between master and slave + :rtype: str + """ - log.info("Executing: %s" % query) + LOG.info("Executing: %s", query) master_f, master_filename = tempfile.mkstemp(prefix="master.") slave_f, slave_filename = tempfile.mkstemp(prefix="slave.") @@ -283,7 +327,7 @@ def print_horizontal(cur_master, cur_slave, query): os.write(master_f, "\n") os.close(master_f) - log.info("Executing: %s" % query) + LOG.info("Executing: %s", query) cur_slave.execute(query) result = cur_slave.fetchall() for row in result: @@ -298,90 +342,123 @@ def print_horizontal(cur_master, cur_slave, query): os.close(slave_f) diffs = diff(open(master_filename).readlines(), - open(slave_filename).readlines()) + open(slave_filename).readlines(), + color=color) os.remove(master_filename) os.remove(slave_filename) return diffs -def print_vertical(master, slave, user, passwd, query): - log.info("Executing: %s" % query) +def print_vertical(master, slave, user, passwd, query, color=True): + r""" + Find and return differences in vertical format. + The vertical format is when you end MySQL query with '\G' + + :param master: Hostname of the master. + :type master: str + :param slave: Hostname of the slave. + :type slave: str + :param query: Query to find records in a chunk we compare + :type query: str + :param color: If True - produce colorful output + :return: Differences in a chunk between master and slave + :rtype: str + """ + LOG.info("Executing: %s", query) proc = Popen(['mysql', '-h', master, '-u', user, '-p%s' % passwd, - '-e', '%s\G' % query], + '-e', r'%s\G' % query], stdout=PIPE, stderr=PIPE) master_cout, master_cerr = proc.communicate() if proc.returncode: - log.error('Failed to query master.') - log.error(master_cerr) + LOG.error('Failed to query master.') + LOG.error(master_cerr) exit(1) - log.info("Executing: %s" % query) + LOG.info("Executing: %s", query) proc = Popen(['mysql', '-h', slave, '-u', user, '-p%s' % passwd, - '-e', '%s\G' % query], + '-e', r'%s\G' % query], stdout=PIPE, stderr=PIPE) slave_cout, slave_cerr = proc.communicate() if proc.returncode: - log.error('Failed to query slave.') - log.error(slave_cerr) + LOG.error('Failed to query slave.') + LOG.error(slave_cerr) exit(1) - return diff(master_cout.split('\n'), slave_cout.split('\n')) + return diff(master_cout.split('\n'), slave_cout.split('\n'), color=color) def get_inconsistencies(db, tbl, slave, user, passwd, - ch_db='percona', ch_tbl='checksums', vertical=False): - try: - conn_slave = MySQLdb.connect(host=slave, user=user, passwd=passwd) - master = get_master(conn_slave) - conn_master = MySQLdb.connect(host=master, user=user, passwd=passwd) - - # Get chunks that are different on the slave and its master - query = ("SELECT chunk " - "FROM `%s`.`%s` " - "WHERE (this_crc<>master_crc OR this_cnt<>master_cnt) " - "AND db='%s' AND tbl='%s'") - log.info("Executing: %s" % (query % (ch_db, ch_tbl, db, tbl))) - cur_master = conn_master.cursor() - cur_slave = conn_slave.cursor() - - cur_slave.execute(query % (ch_db, ch_tbl, db, tbl)) - chunks = cur_slave.fetchall() - - if len(chunks) == 1: - chunks_str = "chunk" - else: - chunks_str = "chunks" - log.info("Found %d inconsistent %s" % (len(chunks), chunks_str)) - # generate WHERE clause to fetch records of the chunk - for chunk, in chunks: + ch_db='percona', ch_tbl='checksums', + vertical=False, color=True): + r""" + Print differences between slave and its master. + + :param db: Database name of the inconsistent table. + :param tbl: Table name of the inconsistent table. + :param slave: Hostname of the slave. + :param user: User to connect to MySQL. + :param passwd: Password to connect to MySQL. + :param ch_db: Database where checksums are stored. + :param ch_tbl: Table name where checksums are stored. + :param vertical: If True - print result vertically (\G in MySQL) + :param color: If True - print colorful output + """ + conn_slave = MySQLdb.connect(host=slave, user=user, passwd=passwd) + master = get_master(conn_slave) + conn_master = MySQLdb.connect(host=master, user=user, passwd=passwd) + + # Get chunks that are different on the slave and its master + query = ("SELECT chunk " + "FROM `%s`.`%s` " + "WHERE (this_crc<>master_crc OR this_cnt<>master_cnt) " + "AND db='%s' AND tbl='%s'") + LOG.info("Executing: %s", query % (ch_db, ch_tbl, db, tbl)) + cur_master = conn_master.cursor() + cur_slave = conn_slave.cursor() + + cur_slave.execute(query % (ch_db, ch_tbl, db, tbl)) + chunks = cur_slave.fetchall() + + if len(chunks) == 1: + chunks_str = "chunk" + else: + chunks_str = "chunks" + LOG.info("Found %d inconsistent %s", len(chunks), chunks_str) + # generate WHERE clause to fetch records of the chunk + for chunk, in chunks: - query = build_chunk_query(db, tbl, chunk, conn_slave, ch_db=ch_db, - ch_tbl=ch_tbl) + query = build_chunk_query(db, tbl, chunk, conn_slave, ch_db=ch_db, + ch_tbl=ch_tbl) - if vertical: - diffs = print_vertical(master, slave, user, passwd, query) - else: - diffs = print_horizontal(cur_master, cur_slave, query) - log.info("Differences between slave %s and its master:" - % slave) - - print(diffs) + if vertical: + diffs = print_vertical(master, slave, user, passwd, query, + color=color) + else: + diffs = print_horizontal(cur_master, cur_slave, query, color=color) + LOG.info("Differences between slave %s and its master:", slave) - except MySQLdb.Error as err: - log.error(err) + print(diffs) def get_inconsistent_tables(host, user, password, ch_db='percona', ch_tbl='checksums'): - try: - conn = MySQLdb.connect(host=host, user=user, passwd=password) - cur = conn.cursor() - cur.execute("SELECT db, tbl FROM `%s`.`%s` " - "WHERE this_crc <> master_crc OR this_cnt <> master_cnt" - % (ch_db, ch_tbl)) - return cur.fetchall() - except MySQLdb.Error as err: - log.error(err) - return [] + """ + On a given MySQL server find tables that are inconsistent with the master. + + :param host: Hostname with potentially inconsistent tables. + :param user: MySQL user. + :param password: MySQL password. + :param ch_db: Database where checksums are stored. + :param ch_tbl: Table name where checksums are stored. + :return: List of tuples with inconsistent tables. + Each tuple is database name, table name + :rtype: list + """ + conn = MySQLdb.connect(host=host, user=user, passwd=password) + cur = conn.cursor() + cur.execute("SELECT db, tbl FROM `%s`.`%s` " + "WHERE this_crc <> master_crc OR this_cnt <> master_cnt" + % (ch_db, ch_tbl)) + return cur.fetchall()