From 5c159688380bc0d6b8a5a6ad33765bc455f91e29 Mon Sep 17 00:00:00 2001 From: David Zafman Date: Thu, 30 Jun 2016 23:18:03 -0700 Subject: [PATCH] tools: New "removeall" used to remove head with snapshots Use --force to allow remove to only remove head object Adding testing to unit test Signed-off-by: David Zafman --- src/test/ceph_objectstore_tool.py | 54 +++++++++++++++++++++ src/tools/ceph_objectstore_tool.cc | 76 +++++++++++++++++++++++++----- 2 files changed, 117 insertions(+), 13 deletions(-) diff --git a/src/test/ceph_objectstore_tool.py b/src/test/ceph_objectstore_tool.py index d3f7d488080a56..1b9a0a6200c9e7 100755 --- a/src/test/ceph_objectstore_tool.py +++ b/src/test/ceph_objectstore_tool.py @@ -575,6 +575,55 @@ def test_get_set_inc_osdmap(CFSD_PREFIX, osd_path): return errors +def test_removeall(CFSD_PREFIX, db, OBJREPPGS, REP_POOL, CEPH_BIN, OSDDIR): + # Test removeall + errors=0 + print "Test removeall" + kill_daemons() + for nspace in db.keys(): + for basename in db[nspace].keys(): + JSON = db[nspace][basename]['json'] + for pg in OBJREPPGS: + OSDS = get_osds(pg, OSDDIR) + for osd in OSDS: + DIR = os.path.join(OSDDIR, os.path.join(osd, os.path.join("current", "{pg}_head".format(pg=pg)))) + fnames = [f for f in os.listdir(DIR) if os.path.isfile(os.path.join(DIR, f)) + and f.split("_")[0] == basename and f.split("_")[4] == nspace] + if not fnames: + continue + cmd = (CFSD_PREFIX + " --force --dry-run '{json}' remove").format(osd=osd, json=JSON) + logging.debug(cmd) + ret = call(cmd, shell=True) + if ret != 0: + logging.error("remove with --force failed for {json}".format(json=JSON)) + errors += 1 + + cmd = (CFSD_PREFIX + " --dry-run '{json}' removeall").format(osd=osd, json=JSON) + logging.debug(cmd) + ret = call(cmd, shell=True) + if ret != 0: + logging.error("removeall failed for {json}".format(json=JSON)) + errors += 1 + + cmd = (CFSD_PREFIX + " '{json}' removeall").format(osd=osd, json=JSON) + logging.debug(cmd) + ret = call(cmd, shell=True) + if ret != 0: + logging.error("removeall failed for {json}".format(json=JSON)) + errors += 1 + vstart(new=False) + wait_for_health() + cmd = "{path}/rados -p {pool} rmsnap snap1".format(pool=REP_POOL, path=CEPH_BIN) + logging.debug(cmd) + ret = call(cmd, shell=True) + if ret != 0: + logging.error("rados rmsnap failed") + errors += 1 + time.sleep(2) + wait_for_health() + return errors + + def main(argv): sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0) if len(argv) > 1 and argv[1] == "debug": @@ -958,6 +1007,9 @@ def main(argv): cmd = (CFSD_PREFIX + "'[\"1.3\",{{\"snapid\":\"not an int\"}}]' list-omap").format(osd=ONEOSD, pg=ONEPG) ERRORS += test_failure(cmd, "Decode object JSON error: value type is 2 not 4") + cmd = (CFSD_PREFIX + "'[\"1.0\", {{\"hash\": 623868980, \"max\": 0, \"oid\": \"REPobject1\", \"namespace\": \"\", \"snapid\": -2, \"key\": \"\", \"pool\": 1}}]' remove").format(osd=ONEOSD) + ERRORS += test_failure(cmd, "Snapshots are present, use removeall to delete everything") + TMPFILE = r"/tmp/tmp.{pid}".format(pid=pid) ALLPGS = OBJREPPGS + OBJECPGS OSDS = get_osds(ALLPGS[0], OSDDIR) @@ -1862,6 +1914,8 @@ def main(argv): call("/bin/rm -rf {dir}".format(dir=TESTDIR), shell=True) call("/bin/rm -rf {dir}".format(dir=DATADIR), shell=True) + ERRORS += test_removeall(CFSD_PREFIX, db, OBJREPPGS, REP_POOL, CEPH_BIN, OSDDIR) + # vstart() starts 4 OSDs ERRORS += test_get_set_osdmap(CFSD_PREFIX, range(4), ALLOSDS) ERRORS += test_get_set_inc_osdmap(CFSD_PREFIX, ALLOSDS[0]) diff --git a/src/tools/ceph_objectstore_tool.cc b/src/tools/ceph_objectstore_tool.cc index 054a6b4b003b8b..b04b40cb77dc42 100644 --- a/src/tools/ceph_objectstore_tool.cc +++ b/src/tools/ceph_objectstore_tool.cc @@ -1425,8 +1425,25 @@ int do_meta(ObjectStore *store, string object, Formatter *formatter, bool debug, return 0; } +int remove_object(coll_t coll, ghobject_t &ghobj, + SnapMapper &mapper, + MapCacher::Transaction *_t, + ObjectStore::Transaction *t) +{ + int r = mapper.remove_oid(ghobj.hobj, _t); + if (r < 0 && r != -ENOENT) { + cerr << "remove_oid returned " << cpp_strerror(r) << std::endl; + return r; + } + + t->remove(coll, ghobj); + return 0; +} + +int get_snapset(ObjectStore *store, coll_t coll, ghobject_t &ghobj, SnapSet &ss, bool silent); + int do_remove_object(ObjectStore *store, coll_t coll, - ghobject_t &ghobj, + ghobject_t &ghobj, bool all, bool force, ObjectStore::Sequencer &osr) { spg_t pg; @@ -1444,20 +1461,52 @@ int do_remove_object(ObjectStore *store, coll_t coll, return r; } - cout << "remove " << ghobj << std::endl; - if (dry_run) - return 0; + SnapSet ss; + if (ghobj.hobj.has_snapset()) { + r = get_snapset(store, coll, ghobj, ss, false); + if (r < 0) { + cerr << "Can't get snapset error " << cpp_strerror(r) << std::endl; + return r; + } + if (!ss.snaps.empty() && !all) { + if (force) { + cout << "WARNING: only removing " + << (ghobj.hobj.is_head() ? "head" : "snapdir") + << " with snapshots present" << std::endl; + ss.snaps.clear(); + } else { + cerr << "Snapshots are present, use removeall to delete everything" << std::endl; + return -EINVAL; + } + } + } + ObjectStore::Transaction t; OSDriver::OSTransaction _t(driver.get_transaction(&t)); - r = mapper.remove_oid(ghobj.hobj, &_t); - if (r < 0 && r != -ENOENT) { - cerr << "remove_oid returned " << cpp_strerror(r) << std::endl; - return r; + + cout << "remove " << ghobj << std::endl; + + if (!dry_run) { + r = remove_object(coll, ghobj, mapper, &_t, &t); + if (r < 0) + return r; + } + + ghobject_t snapobj = ghobj; + for (vector::iterator i = ss.snaps.begin() ; + i != ss.snaps.end() ; ++i) { + snapobj.hobj.snap = *i; + cout << "remove " << snapobj << std::endl; + if (!dry_run) { + r = remove_object(coll, snapobj, mapper, &_t, &t); + if (r < 0) + return r; + } } - t.remove(coll, ghobj); + if (!dry_run) + store->apply_transaction(&osr, std::move(t)); - store->apply_transaction(&osr, std::move(t)); return 0; } @@ -2156,7 +2205,7 @@ void usage(po::options_description &desc) cerr << "ceph-objectstore-tool ... set-omaphdr [file]" << std::endl; cerr << "ceph-objectstore-tool ... list-attrs" << std::endl; cerr << "ceph-objectstore-tool ... list-omap" << std::endl; - cerr << "ceph-objectstore-tool ... remove" << std::endl; + cerr << "ceph-objectstore-tool ... remove|removeall" << std::endl; cerr << "ceph-objectstore-tool ... dump" << std::endl; cerr << "ceph-objectstore-tool ... set-size" << std::endl; cerr << "ceph-objectstore-tool ... remove-clone-metadata " << std::endl; @@ -2828,8 +2877,9 @@ int main(int argc, char **argv) if (vm.count("objcmd")) { ret = 0; - if (objcmd == "remove") { - ret = do_remove_object(fs, coll, ghobj, *osr); + if (objcmd == "remove" || objcmd == "removeall") { + bool all = (objcmd == "removeall"); + ret = do_remove_object(fs, coll, ghobj, all, force, *osr); goto out; } else if (objcmd == "list-attrs") { ret = do_list_attrs(fs, coll, ghobj);