/
sieve.py
159 lines (146 loc) · 6.08 KB
/
sieve.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
"""
dbsake.cmd.sieve
~~~~~~~~~~~~~~~~
Advanced filtering of mysqldump backup files
"""
import errno
import signal
import sys
import click
from dbsake.cli import dbsake
from dbsake.util import compression
@dbsake.command('sieve', options_metavar='[options]')
@click.option('-F', '--format', 'output_format',
metavar='<name>',
default='stream',
type=click.Choice(['stream', 'directory']),
help="Select the output format (directory, stream)")
@click.option('-C', '--directory',
default='.',
metavar='<path>',
type=click.Path(resolve_path=True),
help="Specify output directory when --format=directory")
@click.option('-i', '--input-file',
metavar='<path>',
type=click.File(mode='rb', lazy=False),
default='-',
help="Specify input file to process instead of stdin")
@click.option('-z', '--compress-command',
metavar='<name>',
default=compression.filetype_to_command('.gz'),
help="Specify compression command when --format=directory")
@click.option('-t', '--table',
metavar='<glob>',
multiple=True,
help="Only output tables matching the given glob pattern")
@click.option('-T', '--exclude-table',
metavar='<glob>',
multiple=True,
help="Excludes tables matching the given glob pattern")
@click.option('--defer-indexes',
is_flag=True,
help="Add secondary indexes after loading table data")
@click.option('--defer-foreign-keys',
is_flag=True,
help="Add foreign key constraints after loading table data")
@click.option('--write-binlog/--no-write-binlog',
default=True,
help="Include SQL_LOG_BIN = 0 in output to disable binlog")
@click.option('--table-schema/--no-table-schema',
default=True,
help="Include/exclude table schema from output.")
@click.option('--table-data/--no-table-data',
default=True,
help="Include/exclude table data from output")
@click.option('--routines/--no-routines',
default=None,
help="Include / exclude database routines from output")
@click.option('--events/--no-events',
default=None,
help="Include / exclude database events from output")
@click.option('--triggers/--no-triggers',
default=None,
help="Include/exclude table triggers from output")
@click.option('--master-data/--no-master-data',
default=None,
help="Uncomment/comment CHANGE MASTER in input, if present")
@click.option('-O', '--to-stdout', is_flag=True,
help="Force output on stdout, even to a terminal.")
@click.pass_context
def sieve_cli(ctx,
output_format,
directory,
input_file,
compress_command,
table,
exclude_table,
defer_indexes,
defer_foreign_keys,
write_binlog,
table_schema,
table_data,
routines,
events,
triggers,
master_data,
to_stdout):
"""Filter and transform mysqldump output.
sieve can extract single tables from a mysqldump file and perform useful
transformations, such as adding indexes after the table data is loaded
for InnoDB tables, where such indexes can be created much more efficiently
than the default incremental rebuild that mysqldump performs.
Example:
$ dbsake sieve --no-table-data < sakila.sql.gz > sakila_schema.sql
$ dbsake sieve --format=directory -i sakila.sql.gz -C extracted_sql/
"""
from dbsake.core.mysql import sieve
if hasattr(input_file, 'detach'):
input_file = input_file.detach()
if output_format == 'stream' and sys.stdout.isatty() and not to_stdout:
ctx.fail("stdout appears to be a terminal and --format=stream. "
"Use -O/--to-stdout to force output or redirect to a file. "
"Aborting.")
if defer_indexes and not table_data:
click.echo("Disabling index deferment since --no-data requested",
file=sys.stderr)
defer_indexes = False
defer_foreign_keys = False
options = sieve.Options(output_format=output_format,
table_schema=table_schema,
table_data=table_data,
routines=routines,
events=events,
triggers=triggers,
master_data=master_data,
defer_indexes=defer_indexes,
defer_foreign_keys=defer_foreign_keys,
table=table,
exclude_table=exclude_table,
write_binlog=write_binlog,
directory=directory,
compress_command=compress_command,
input_stream=input_file,
output_stream=click.get_binary_stream('stdout'))
try:
stats = sieve.sieve(options)
except IOError as exc:
if exc.errno != errno.EPIPE:
raise # generate a traceback, in case this is a bug
else:
# note broken pipe in debug mode
if ctx.obj['debug']:
click.echo("Broken pipe (errno: %d)" % exc.errno,
file=sys.stderr)
# exit with SIGPIPE to indicate only partial output
sys.exit(128 + signal.SIGPIPE)
except sieve.Error as exc:
click.echo(exc, file=sys.stderr)
sys.exit(1)
else:
click.echo(("Processed %s. "
"Output: %d database(s) %d table(s) and %d view(s)") %
(options.input_stream.name,
stats['createdatabase'] or 1,
stats['tablestructure'],
stats['view']), file=sys.stderr)
sys.exit(0)