Skip to content

Commit

Permalink
osd: load classes from a fixed location off disk
Browse files Browse the repository at this point in the history 
Simplify the class loading code to just load out of a local directory.
Do not request classes from the monitor.

Signed-off-by: Sage Weil <sage.weil@dreamhost.com>
  • Loading branch information
Sage Weil committed Apr 19, 2011
1 parent 2e41f08 commit 964a0a6
Show file tree
Hide file tree
Showing 9 changed files with 166 additions and 497 deletions.
408 changes: 93 additions & 315 deletions src/common/ClassHandler.cc
View file

Large diffs are not rendered by default.

74 changes: 27 additions & 47 deletions src/common/ClassHandler.h
View file
Expand Up @@ -11,15 +11,8 @@
#include "common/ClassVersion.h"


class OSD;
class MClass;



class ClassHandler
{
OSD *osd;

public:
class ClassData;

Expand All @@ -37,68 +30,55 @@ class ClassHandler
};

struct ClassData {
Mutex *mutex;
bool _add_dependency(cls_deps_t *dep);
void _add_dependent(ClassData& dependent);

void satisfy_dependency(ClassData *cls);

enum Status {
CLASS_UNKNOWN,
CLASS_INVALID,
CLASS_ERROR,
CLASS_LOADED,
CLASS_REQUESTED,
CLASS_MISSING, // missing
CLASS_MISSING_DEPS, // missing dependencies
CLASS_INITIALIZING, // calling init() right now
CLASS_OPEN, // initialized, usable
} status;
ClassVersion version;
utime_t expires;
ClassImpl impl;

string name;
OSD *osd;
ClassHandler *handler;
void *handle;
bool registered;

map<string, ClassMethod> methods_map;

map<string, ClassData *> dependencies; /* our dependencies */
map<string, ClassData *> missing_dependencies; /* only missing dependencies */
list<ClassData *> dependents; /* classes that depend on us */
set<ClassData *> dependencies; /* our dependencies */
set<ClassData *> missing_dependencies; /* only missing dependencies */

bool has_missing_deps() { return (missing_dependencies.size() > 0); }
ClassMethod *_get_method(const char *mname);

ClassData() : mutex(NULL), status(CLASS_UNKNOWN), version(), handle(NULL), registered(false) {}
~ClassData() { if (mutex) delete mutex; }
ClassData() : status(CLASS_UNKNOWN),
handle(NULL) {}
~ClassData() { }

ClassMethod *register_method(const char *mname, int flags, cls_method_call_t func);
ClassMethod *register_cxx_method(const char *mname, int flags, cls_method_cxx_call_t func);
ClassMethod *get_method(const char *mname);
void unregister_method(ClassMethod *method);

int load();
void init();

void set_status(Status _status);
bool cache_timed_out();
ClassMethod *get_method(const char *mname) {
Mutex::Locker l(handler->mutex);
return _get_method(mname);
}
int get_method_flags(const char *mname);
};

private:
Mutex mutex;
map<string, ClassData> classes;

ClassData& get_obj(const string& cname);

int load_class(const string& cname);
int _load_class(ClassData &data);

ClassHandler(OSD *_osd) : osd(_osd), mutex("ClassHandler") {}

ClassData *get_class(const string& cname, ClassVersion& version);
void resend_class_requests();

void handle_class(MClass *m);
ClassData *_get_class(const string& cname);
int _load_class(ClassData *cls);

public:
ClassHandler() : mutex("ClassHandler") {}

int open_class(const string& cname, ClassData **pcls);

ClassData *register_class(const char *cname);
void unregister_class(ClassData *cls);

int get_method_flags(const string& cname, const string& mname);

};


Expand Down
2 changes: 1 addition & 1 deletion src/common/config.cc
View file
Expand Up @@ -368,7 +368,7 @@ struct config_option config_optionsp[] = {
OPTION(osd_auto_weight, OPT_BOOL, false),
OPTION(osd_class_error_timeout, OPT_DOUBLE, 60.0), // seconds
OPTION(osd_class_timeout, OPT_DOUBLE, 60*60.0), // seconds
OPTION(osd_class_tmp, OPT_STR, "/var/lib/ceph/tmp"),
OPTION(osd_class_dir, OPT_STR, "/usr/lib/rados-classes"),
OPTION(osd_check_for_log_corruption, OPT_BOOL, false),
OPTION(osd_use_stale_snap, OPT_BOOL, false),
OPTION(osd_max_notify_timeout, OPT_U32, 30), // max notify timeout in seconds
Expand Down
2 changes: 1 addition & 1 deletion src/common/config.h
View file
Expand Up @@ -416,7 +416,7 @@ struct md_config_t

double osd_class_error_timeout;
double osd_class_timeout;
std::string osd_class_tmp;
std::string osd_class_dir;

int osd_max_scrubs;
float osd_scrub_load_threshold;
Expand Down
40 changes: 13 additions & 27 deletions src/objclass/class_api.cc
View file
Expand Up @@ -2,21 +2,20 @@
#include "common/config.h"

#include "objclass/objclass.h"
#include "osd/OSD.h"
#include "osd/ReplicatedPG.h"

#include "common/ClassHandler.h"

static OSD *osd;
static ClassHandler *ch;

void cls_initialize(OSD *_osd)
void cls_initialize(ClassHandler *h)
{
osd = _osd;
ch = h;
}

void cls_finalize()
{
osd = NULL;
ch = NULL;
}


Expand All @@ -32,33 +31,24 @@ void cls_free(void *p)

int cls_register(const char *name, cls_handle_t *handle)
{
ClassHandler::ClassData *cd;

cd = osd->class_handler->register_class(name);

*handle = (cls_handle_t)cd;

return (cd != NULL);
ClassHandler::ClassData *cls = ch->register_class(name);
*handle = (cls_handle_t)cls;
return (cls != NULL);
}

int cls_unregister(cls_handle_t handle)
{
ClassHandler::ClassData *cd;
cd = (ClassHandler::ClassData *)handle;

osd->class_handler->unregister_class(cd);
ClassHandler::ClassData *cls = (ClassHandler::ClassData *)handle;
ch->unregister_class(cls);
return 1;
}

int cls_register_method(cls_handle_t hclass, const char *method,
int flags,
cls_method_call_t class_call, cls_method_handle_t *handle)
{
ClassHandler::ClassData *cd;
cls_method_handle_t hmethod;

cd = (ClassHandler::ClassData *)hclass;
hmethod = (cls_method_handle_t)cd->register_method(method, flags, class_call);
ClassHandler::ClassData *cls = (ClassHandler::ClassData *)hclass;
cls_method_handle_t hmethod =(cls_method_handle_t)cls->register_method(method, flags, class_call);
if (handle)
*handle = hmethod;
return (hmethod != NULL);
Expand All @@ -68,11 +58,8 @@ int cls_register_cxx_method(cls_handle_t hclass, const char *method,
int flags,
cls_method_cxx_call_t class_call, cls_method_handle_t *handle)
{
ClassHandler::ClassData *cd;
cls_method_handle_t hmethod;

cd = (ClassHandler::ClassData *)hclass;
hmethod = (cls_method_handle_t)cd->register_cxx_method(method, flags, class_call);
ClassHandler::ClassData *cls = (ClassHandler::ClassData *)hclass;
cls_method_handle_t hmethod = (cls_method_handle_t)cls->register_cxx_method(method, flags, class_call);
if (handle)
*handle = hmethod;
return (hmethod != NULL);
Expand All @@ -82,7 +69,6 @@ int cls_unregister_method(cls_method_handle_t handle)
{
ClassHandler::ClassMethod *method = (ClassHandler::ClassMethod *)handle;
method->unregister();

return 1;
}

Expand Down
100 changes: 17 additions & 83 deletions src/osd/OSD.cc
View file
Expand Up @@ -454,7 +454,6 @@ OSD::OSD(int id, Messenger *internal_messenger, Messenger *external_messenger,
osdmap(NULL),
map_lock("OSD::map_lock"),
map_cache_lock("OSD::map_cache_lock"), map_cache_keep_from(0),
class_lock("OSD::class_lock"),
up_thru_wanted(0), up_thru_pending(0),
pg_stat_queue_lock("OSD::pg_stat_queue_lock"),
osd_stat_updated(false),
Expand Down Expand Up @@ -514,7 +513,7 @@ void handle_signal(int signal)
}
}

void cls_initialize(OSD *_osd);
void cls_initialize(ClassHandler *ch);


int OSD::pre_init()
Expand Down Expand Up @@ -568,10 +567,10 @@ int OSD::init()
return -1;
}

class_handler = new ClassHandler(this);
class_handler = new ClassHandler();
if (!class_handler)
return -ENOMEM;
cls_initialize(this);
cls_initialize(class_handler);

// load up "current" osdmap
assert_warn(!osdmap);
Expand Down Expand Up @@ -1833,7 +1832,6 @@ void OSD::ms_handle_connect(Connection *con)
send_pg_temp();
send_failures();
send_pg_stats(g_clock.now());
class_handler->resend_class_requests();
}
}

Expand Down Expand Up @@ -2553,10 +2551,6 @@ void OSD::_dispatch(Message *m)
handle_rep_scrub((MOSDRepScrub*)m);
break;

case MSG_CLASS:
handle_class((MClass*)m);
break;

// -- need OSDMap --

default:
Expand Down Expand Up @@ -5110,7 +5104,11 @@ void OSD::handle_op(MOSDOp *op)

utime_t now = g_clock.now();

init_op_flags(op);
int r = init_op_flags(op);
if (r) {
reply_op_error(op, r);
return;
}

// calc actual pgid
pg_t pgid = op->get_pg();
Expand Down Expand Up @@ -5471,78 +5469,7 @@ void OSD::wait_for_no_ops()

// --------------------------------

int OSD::get_class(const string& cname, ClassVersion& version, pg_t pgid, Message *m,
ClassHandler::ClassData **pcls)
{
Mutex::Locker l(class_lock);
dout(10) << "get_class '" << cname << "' by " << m << dendl;

ClassHandler::ClassData *cls = class_handler->get_class(cname, version);
if (cls) {
switch (cls->status) {
case ClassHandler::ClassData::CLASS_LOADED:
*pcls = cls;
return 0;
case ClassHandler::ClassData::CLASS_INVALID:
dout(1) << "class " << cname << " not supported" << dendl;
return -EOPNOTSUPP;
case ClassHandler::ClassData::CLASS_ERROR:
dout(0) << "error loading class!" << dendl;
return -EIO;
default:
assert(0);
}
}

dout(10) << "get_class '" << cname << "' by " << m << " waiting for missing class" << dendl;
waiting_for_missing_class[cname].push_back(m);
return -EAGAIN;
}

void OSD::got_class(const string& cname)
{
// no lock.. this is an upcall from handle_class
dout(10) << "got_class '" << cname << "'" << dendl;

if (waiting_for_missing_class.count(cname)) {
take_waiters(waiting_for_missing_class[cname]);
waiting_for_missing_class.erase(cname);
}
}

void OSD::handle_class(MClass *m)
{
if (!require_mon_peer(m))
return;

Mutex::Locker l(class_lock);
dout(10) << "handle_class action=" << m->action << dendl;

switch (m->action) {
case CLASS_RESPONSE:
class_handler->handle_class(m);
break;

default:
assert(0);
}
m->put();
}

void OSD::send_class_request(const char *cname, ClassVersion& version)
{
dout(10) << "send_class_request class=" << cname << " version=" << version << dendl;
MClass *m = new MClass(monc->get_fsid(), 0);
ClassInfo info;
info.name = cname;
info.version = version;
m->info.push_back(info);
m->action = CLASS_GET;
monc->send_mon_message(m);
}


void OSD::init_op_flags(MOSDOp *op)
int OSD::init_op_flags(MOSDOp *op)
{
vector<OSDOp>::iterator iter;

Expand All @@ -5568,7 +5495,12 @@ void OSD::init_op_flags(MOSDOp *op)
string cname, mname;
bp.copy(iter->op.cls.class_len, cname);
bp.copy(iter->op.cls.method_len, mname);
int flags = class_handler->get_method_flags(cname, mname);

ClassHandler::ClassData *cls;
int r = class_handler->open_class(cname, &cls);
if (r)
return r;
int flags = cls->get_method_flags(mname.c_str());
is_read = flags & CLS_METHOD_RD;
is_write = flags & CLS_METHOD_WR;
is_public = flags & CLS_METHOD_PUBLIC;
Expand All @@ -5590,4 +5522,6 @@ void OSD::init_op_flags(MOSDOp *op)
break;
}
}

return 0;
}

0 comments on commit 964a0a6

Please sign in to comment.