Merge 91b2aca into 37901af

ceph.spec.in

@@ -706,8 +706,7 @@ ln -sf %{_libdir}/librbd.so.1 /usr/lib64/qemu/librbd.so.1
 %{_bindir}/ceph_smalliobenchdumb
 %{_bindir}/ceph_smalliobenchfs
 %{_bindir}/ceph_smalliobenchrbd
-%{_bindir}/ceph_filestore_dump
-%{_bindir}/ceph_filestore_tool
+%{_bindir}/ceph-objectstore-tool
 %{_bindir}/ceph_streamtest
 %{_bindir}/ceph_test_*
 %{_bindir}/ceph_tpbench

debian/ceph-test.install

@@ -1,8 +1,7 @@
 usr/bin/ceph-coverage
 usr/bin/ceph_bench_log
 usr/bin/ceph_dupstore
-usr/bin/ceph_filestore_dump
-usr/bin/ceph_filestore_tool
+usr/bin/ceph-objectstore-tool
 usr/bin/ceph_kvstorebench
 usr/bin/ceph_multi_stress_watch
 usr/bin/ceph_erasure_code

src/.gitignore

@@ -24,8 +24,7 @@ Makefile
 /ceph.conf
 /ceph_bench_log
 /ceph_dupstore
-/ceph_filestore_dump
-/ceph_filestore_tool
+/ceph-objectstore-tool
 /ceph_mon_store_converter
 /ceph_multi_stress_watch
 /ceph_erasure_code

src/common/code_environment.h

@@ -19,6 +19,7 @@ enum code_environment_t {
   CODE_ENVIRONMENT_UTILITY = 0,
   CODE_ENVIRONMENT_DAEMON = 1,
   CODE_ENVIRONMENT_LIBRARY = 2,
+  CODE_ENVIRONMENT_UTILITY_NODOUT = 3,
 };
 
 #ifdef __cplusplus

src/common/common_init.cc

@@ -63,6 +63,7 @@ CephContext *common_preinit(const CephInitParameters &iparams,
       conf->set_val("keyring", "$osd_data/keyring", false);
     break;
 
+  case CODE_ENVIRONMENT_UTILITY_NODOUT:
   case CODE_ENVIRONMENT_LIBRARY:
     conf->set_val_or_die("log_to_stderr", "false");
     conf->set_val_or_die("err_to_stderr", "false");

src/common/hobject.cc

@@ -235,6 +235,20 @@ void ghobject_t::decode(bufferlist::iterator& bl)
   DECODE_FINISH(bl);
 }
 
+void ghobject_t::decode(json_spirit::Value& v)
+{
+  hobj.decode(v);
+  using namespace json_spirit;
+  Object& o = v.get_obj();
+  for (Object::size_type i=0; i<o.size(); i++) {
+    Pair& p = o[i];
+    if (p.name_ == "generation")
+      generation = p.value_.get_uint64();
+    else if (p.name_ == "shard_id")
+      shard_id = p.value_.get_int();
+  }
+}
+
 void ghobject_t::dump(Formatter *f) const
 {
   hobj.dump(f);

src/os/FileStore.cc

@@ -419,10 +419,11 @@ int FileStore::lfn_unlink(coll_t cid, const ghobject_t& o,
   return index->unlink(o);
 }
 
-FileStore::FileStore(const std::string &base, const std::string &jdev, const char *name, bool do_update) :
+FileStore::FileStore(const std::string &base, const std::string &jdev, osflagbits_t flags, const char *name, bool do_update) :
   JournalingObjectStore(base),
   internal_name(name),
   basedir(base), journalpath(jdev),
+  generic_flags(flags),
   blk_size(0),
   fsid_fd(-1), op_fd(-1),
   basedir_fd(-1), current_fd(-1),
@@ -1122,7 +1123,7 @@ int FileStore::write_op_seq(int fd, uint64_t seq)
   return ret;
 }
 
-int FileStore::mount() 
+int FileStore::mount()
 {
   int ret;
   char buf[PATH_MAX];
@@ -1350,7 +1351,7 @@ int FileStore::mount()
     ::unlink(nosnapfn);
   }
 
-  {
+  if (!(generic_flags & SKIP_MOUNT_OMAP)) {
     LevelDBStore *omap_store = new LevelDBStore(g_ceph_context, omap_dir);
 
     omap_store->init();
@@ -1462,24 +1463,26 @@ int FileStore::mount()
   wbthrottle.start();
   sync_thread.create();
 
-  ret = journal_replay(initial_op_seq);
-  if (ret < 0) {
-    derr << "mount failed to open journal " << journalpath << ": " << cpp_strerror(ret) << dendl;
-    if (ret == -ENOTTY) {
-      derr << "maybe journal is not pointing to a block device and its size "
-	   << "wasn't configured?" << dendl;
-    }
+  if (!(generic_flags & SKIP_JOURNAL_REPLAY)) {
+    ret = journal_replay(initial_op_seq);
+    if (ret < 0) {
+      derr << "mount failed to open journal " << journalpath << ": " << cpp_strerror(ret) << dendl;
+      if (ret == -ENOTTY) {
+        derr << "maybe journal is not pointing to a block device and its size "
+	     << "wasn't configured?" << dendl;
+      }
 
-    // stop sync thread
-    lock.Lock();
-    stop = true;
-    sync_cond.Signal();
-    lock.Unlock();
-    sync_thread.join();
+      // stop sync thread
+      lock.Lock();
+      stop = true;
+      sync_cond.Signal();
+      lock.Unlock();
+      sync_thread.join();
 
-    wbthrottle.stop();
+      wbthrottle.stop();
 
-    goto close_current_fd;
+      goto close_current_fd;
+    }
   }
 
   {
@@ -1532,6 +1535,8 @@ int FileStore::umount()
   op_tp.stop();
 
   journal_stop();
+  if (!(generic_flags & SKIP_JOURNAL_REPLAY))
+    journal_write_close();
 
   op_finisher.stop();
   ondisk_finisher.stop();

src/os/FileStore.h

@@ -126,6 +126,7 @@ class FileStore : public JournalingObjectStore,
 private:
   string internal_name;         ///< internal name, used to name the perfcounter instance
   string basedir, journalpath;
+  osflagbits_t generic_flags;
   std::string current_fn;
   std::string current_op_seq_fn;
   std::string omap_dir;
@@ -405,7 +406,9 @@ class FileStore : public JournalingObjectStore,
 		 bool force_clear_omap=false);
 
 public:
-  FileStore(const std::string &base, const std::string &jdev, const char *internal_name = "filestore", bool update_to=false);
+  FileStore(const std::string &base, const std::string &jdev,
+    osflagbits_t flags = 0,
+    const char *internal_name = "filestore", bool update_to=false);
   ~FileStore();
 
   int _detect_fs();

src/os/JournalingObjectStore.cc

@@ -21,6 +21,11 @@ void JournalingObjectStore::journal_stop()
 {
   dout(10) << "journal_stop" << dendl;
   finisher.stop();
+}
+
+// A journal_replay() makes journal writeable, this closes that out.
+void JournalingObjectStore::journal_write_close()
+{
   if (journal) {
     journal->close();
     delete journal;

src/os/JournalingObjectStore.h

@@ -111,6 +111,7 @@ class JournalingObjectStore : public ObjectStore {
 protected:
   void journal_start();
   void journal_stop();
+  void journal_write_close();
   int journal_replay(uint64_t fs_op_seq);
 
   void _op_journal_transactions(list<ObjectStore::Transaction*>& tls, uint64_t op,

src/os/ObjectStore.cc

@@ -24,10 +24,11 @@
 ObjectStore *ObjectStore::create(CephContext *cct,
 				 const string& type,
 				 const string& data,
-				 const string& journal)
+				 const string& journal,
+			         osflagbits_t flags)
 {
   if (type == "filestore") {
-    return new FileStore(data, journal);
+    return new FileStore(data, journal, flags);
   }
   if (type == "memstore") {
     return new MemStore(cct, data);

src/os/ObjectStore.h

@@ -80,6 +80,11 @@ static inline void encode(const map<string,bufferptr> *attrset, bufferlist &bl)
   ::encode(*attrset, bl);
 }
 
+// Flag bits
+typedef uint32_t osflagbits_t;
+const int SKIP_JOURNAL_REPLAY = 1 << 0;
+const int SKIP_MOUNT_OMAP = 1 << 1;
+
 class ObjectStore {
 protected:
   string path;
@@ -93,11 +98,13 @@ class ObjectStore {
    * @param type type of store. This is a string from the configuration file.
    * @param data path (or other descriptor) for data
    * @param journal path (or other descriptor) for journal (optional)
+   * @param flags which filestores should check if applicable
    */
   static ObjectStore *create(CephContext *cct,
 			     const string& type,
 			     const string& data,
-			     const string& journal);
+			     const string& journal,
+			     osflagbits_t flag = 0);
 
   Logger *logger;
 

src/osd/OSD.cc

@@ -2208,7 +2208,7 @@ void OSD::build_past_intervals_parallel()
 {
   map<PG*,pistate> pis;
 
-  // calculate untion of map range
+  // calculate junction of map range
   epoch_t end_epoch = superblock.oldest_map;
   epoch_t cur_epoch = superblock.newest_map;
   for (ceph::unordered_map<spg_t, PG*>::iterator i = pg_map.begin();
@@ -2217,7 +2217,7 @@ void OSD::build_past_intervals_parallel()
     PG *pg = i->second;
 
     epoch_t start, end;
-    if (!pg->_calc_past_interval_range(&start, &end))
+    if (!pg->_calc_past_interval_range(&start, &end, superblock.oldest_map))
       continue;
 
     dout(10) << pg->info.pgid << " needs " << start << "-" << end << dendl;
@@ -2255,8 +2255,11 @@ void OSD::build_past_intervals_parallel()
       vector<int> acting, up;
       int up_primary;
       int primary;
+      pg_t pgid = pg->info.pgid.pgid;
+      if (p.same_interval_since && last_map->get_pools().count(pgid.pool()))
+	pgid = pgid.get_ancestor(last_map->get_pg_num(pgid.pool()));
       cur_map->pg_to_up_acting_osds(
-	pg->info.pgid.pgid, &up, &up_primary, &acting, &primary);
+	pgid, &up, &up_primary, &acting, &primary);
 
       if (p.same_interval_since == 0) {
 	dout(10) << __func__ << " epoch " << cur_epoch << " pg " << pg->info.pgid
@@ -2283,7 +2286,7 @@ void OSD::build_past_intervals_parallel()
 	pg->info.history.last_epoch_clean,
 	cur_map, last_map,
 	pg->info.pgid.pool(),
-	pg->info.pgid.pgid,
+	pgid,
 	&pg->past_intervals,
 	&debug);
       if (new_interval) {

src/osd/PG.cc

@@ -603,7 +603,7 @@ bool PG::needs_backfill() const
   return ret;
 }
 
-bool PG::_calc_past_interval_range(epoch_t *start, epoch_t *end)
+bool PG::_calc_past_interval_range(epoch_t *start, epoch_t *end, epoch_t oldest_map)
 {
   *end = info.history.same_interval_since;
 
@@ -620,7 +620,7 @@ bool PG::_calc_past_interval_range(epoch_t *start, epoch_t *end)
 
   *start = MAX(MAX(info.history.epoch_created,
 		   info.history.last_epoch_clean),
-	       osd->get_superblock().oldest_map);
+	       oldest_map);
   if (*start >= *end) {
     dout(10) << __func__ << " start epoch " << *start << " >= end epoch " << *end
 	     << ", nothing to do" << dendl;
@@ -634,7 +634,8 @@ bool PG::_calc_past_interval_range(epoch_t *start, epoch_t *end)
 void PG::generate_past_intervals()
 {
   epoch_t cur_epoch, end_epoch;
-  if (!_calc_past_interval_range(&cur_epoch, &end_epoch)) {
+  if (!_calc_past_interval_range(&cur_epoch, &end_epoch,
+      osd->get_superblock().oldest_map)) {
     return;
   }
 
@@ -659,8 +660,8 @@ void PG::generate_past_intervals()
 
     cur_map = osd->get_map(cur_epoch);
     pg_t pgid = get_pgid().pgid;
-    if (cur_map->get_pools().count(pgid.pool()))
-      pgid = pgid.get_ancestor(cur_map->get_pg_num(pgid.pool()));
+    if (last_map->get_pools().count(pgid.pool()))
+      pgid = pgid.get_ancestor(last_map->get_pg_num(pgid.pool()));
     cur_map->pg_to_up_acting_osds(pgid, &up, &up_primary, &acting, &primary);
 
     std::stringstream debug;

src/osd/PG.h

@@ -779,7 +779,7 @@ class PG {
 
   void mark_clean();  ///< mark an active pg clean
 
-  bool _calc_past_interval_range(epoch_t *start, epoch_t *end);
+  bool _calc_past_interval_range(epoch_t *start, epoch_t *end, epoch_t oldest_map);
   void generate_past_intervals();
   void trim_past_intervals();
   void build_prior(std::auto_ptr<PriorSet> &prior_set);

src/osd/ReplicatedPG.cc

@@ -8616,8 +8616,19 @@ int ReplicatedBackend::build_push_op(const ObjectRecoveryInfo &recovery_info,
       dout(10) << " extent " << p.get_start() << "~" << p.get_len()
 	       << " is actually " << p.get_start() << "~" << bit.length()
 	       << dendl;
-      p.set_len(bit.length());
+      interval_set<uint64_t>::iterator save = p++;
+      if (bit.length() == 0)
+        out_op->data_included.erase(save);     //Remove this empty interval
+      else
+        save.set_len(bit.length());
+      // Remove any other intervals present
+      while (p != out_op->data_included.end()) {
+        interval_set<uint64_t>::iterator save = p++;
+        out_op->data_included.erase(save);
+      }
       new_progress.data_complete = true;
+      out_op->data.claim_append(bit);
+      break;
     }
     out_op->data.claim_append(bit);
   }