Skip to content

allowing for configurable data dump directory #26

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Oct 26, 2016
Merged

allowing for configurable data dump directory #26

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,8 @@ optional arguments:
--skipThroughputUpdate
Skip updating throughput values across tables
[optional]
--dataDump Directory to place and search for DynamoDB table
backups (defaults to use 'dump') [optional]
--log LOG Logging level - DEBUG|INFO|WARNING|ERROR|CRITICAL
[optional]

Expand Down
27 changes: 14 additions & 13 deletions dynamodump.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
MAX_RETRY = 6
LOCAL_REGION = "local"
LOG_LEVEL = "INFO"
DUMP_PATH = "dump"
DATA_DUMP = "dump"
RESTORE_WRITE_CAPACITY = 25
THREAD_START_DELAY = 1 # seconds
CURRENT_WORKING_DIR = os.getcwd()
Expand Down Expand Up @@ -61,9 +61,9 @@ def get_table_name_matches(conn, table_name_wildcard, separator):
def get_restore_table_matches(table_name_wildcard, separator):
matching_tables = []
try:
dir_list = os.listdir("./" + DUMP_PATH)
dir_list = os.listdir("./" + args.dumpPath)
except OSError:
logging.info("Cannot find \"./%s\", Now trying current working directory.." % DUMP_PATH)
logging.info("Cannot find \"./%s\", Now trying current working directory.." % args.dumpPath)
dump_data_path = CURRENT_WORKING_DIR
try:
dir_list = os.listdir(dump_data_path)
Expand Down Expand Up @@ -249,13 +249,13 @@ def do_backup(conn, table_name, read_capacity):
logging.info("Starting backup for " + table_name + "..")

# trash data, re-create subdir
if os.path.exists(DUMP_PATH + "/" + table_name):
shutil.rmtree(DUMP_PATH + "/" + table_name)
mkdir_p(DUMP_PATH + "/" + table_name)
if os.path.exists(args.dumpPath + "/" + table_name):
shutil.rmtree(args.dumpPath + "/" + table_name)
mkdir_p(args.dumpPath + "/" + table_name)

# get table schema
logging.info("Dumping table schema for " + table_name)
f = open(DUMP_PATH + "/" + table_name + "/" + SCHEMA_FILE, "w+")
f = open(args.dumpPath + "/" + table_name + "/" + SCHEMA_FILE, "w+")
table_desc = conn.describe_table(table_name)
f.write(json.dumps(table_desc, indent=JSON_INDENT))
f.close()
Expand All @@ -270,15 +270,15 @@ def do_backup(conn, table_name, read_capacity):

# get table data
logging.info("Dumping table items for " + table_name)
mkdir_p(DUMP_PATH + "/" + table_name + "/" + DATA_DIR)
mkdir_p(args.dumpPath + "/" + table_name + "/" + DATA_DIR)

i = 1
last_evaluated_key = None

while True:
scanned_table = conn.scan(table_name, exclusive_start_key=last_evaluated_key)

f = open(DUMP_PATH + "/" + table_name + "/" + DATA_DIR + "/" + str(i).zfill(4) + ".json", "w+")
f = open(args.dumpPath + "/" + table_name + "/" + DATA_DIR + "/" + str(i).zfill(4) + ".json", "w+")
f.write(json.dumps(scanned_table, indent=JSON_INDENT))
f.close()

Expand All @@ -302,10 +302,10 @@ def do_restore(conn, sleep_interval, source_table, destination_table, write_capa

# create table using schema
# restore source_table from dump directory if it exists else try current working directory
if os.path.exists("%s/%s" % (DUMP_PATH, source_table)):
dump_data_path = DUMP_PATH
if os.path.exists("%s/%s" % (args.dumpPath, source_table)):
dump_data_path = args.dumpPath
else:
logging.info("Cannot find \"./%s/%s\", Now trying current working directory.." % (DUMP_PATH, source_table))
logging.info("Cannot find \"./%s/%s\", Now trying current working directory.." % (args.dumpPath, source_table))
if os.path.exists("%s/%s" % (CURRENT_WORKING_DIR, source_table)):
dump_data_path = CURRENT_WORKING_DIR
else:
Expand Down Expand Up @@ -458,6 +458,7 @@ def do_restore(conn, sleep_interval, source_table, destination_table, write_capa
help="Restore data only. Do not delete/recreate schema [optional for restore]")
parser.add_argument("--skipThroughputUpdate", action="store_true", default=False,
help="Skip updating throughput values across tables [optional]")
parser.add_argument("--dumpPath", help="Directory to place and search for DynamoDB table backups (defaults to use '" + str(DATA_DUMP) + "') [optional]", default=str(DATA_DUMP))
parser.add_argument("--log", help="Logging level - DEBUG|INFO|WARNING|ERROR|CRITICAL [optional]")
args = parser.parse_args()

Expand Down Expand Up @@ -542,7 +543,7 @@ def do_restore(conn, sleep_interval, source_table, destination_table, write_capa

matching_restore_tables = get_restore_table_matches(args.srcTable, prefix_separator)
logging.info(
"Found " + str(len(matching_restore_tables)) + " table(s) in " + DUMP_PATH + " to restore: " + ", ".join(
"Found " + str(len(matching_restore_tables)) + " table(s) in " + args.dumpPath + " to restore: " + ", ".join(
matching_restore_tables))

threads = []
Expand Down