Skip to content

Commit

Permalink
Lifecycle testing
Browse files Browse the repository at this point in the history 
Add new config option - rgw_lc_debug_interval - that modifies the
lifecycle algorithm.  It treats days as being the configured number of
seconds.  This allows lifecycle to be tested in minutes, rather than
days.

Signed-off-by: Daniel Gryniewicz <dang@redhat.com>
  • Loading branch information
@dang
dang committed Sep 21, 2016
1 parent 1707da2 commit 77853a4
Show file tree
Hide file tree
Showing 3 changed files with 51 additions and 22 deletions.
1 change: 1 addition & 0 deletions src/common/config_opts.h
View file
Expand Up @@ -1318,6 +1318,7 @@ OPTION(rgw_lifecycle_thread, OPT_INT, 1) //start lifecycle thread number per rad
OPTION(rgw_lifecycle_work_time, OPT_STR, "00:00-06:00") //job process lc at 00:00-06:00s
OPTION(rgw_lc_lock_max_time, OPT_INT, 60) // total run time for a single gc processor work
OPTION(rgw_lc_max_objs, OPT_INT, 32)
OPTION(rgw_lc_debug_interval, OPT_INT, -1) // Debug run interval, in seconds
OPTION(rgw_script_uri, OPT_STR, "") // alternative value for SCRIPT_URI if not set in request
OPTION(rgw_request_uri, OPT_STR, "") // alternative value for REQUEST_URI if not set in request
OPTION(rgw_swift_url, OPT_STR, "") // the swift url, being published by the internal swift auth
Expand Down
66 changes: 46 additions & 20 deletions src/rgw/rgw_lc.cc
View file
Expand Up @@ -58,11 +58,11 @@ void *RGWLC::LCWorker::entry() {
break;

utime_t end = ceph_clock_now(cct);
int secs = shedule_next_start_time(end);
int secs = schedule_next_start_time(start, end);
time_t next_time = end + secs;
char buf[30];
char *nt = ctime_r(&next_time, buf);
dout(5) << "shedule life cycle next start time: " << nt <<dendl;
dout(5) << "schedule life cycle next start time: " << nt <<dendl;

lock.Lock();
cond.WaitInterval(cct, lock, utime_t(secs, 0));
Expand Down Expand Up @@ -105,6 +105,12 @@ bool RGWLC::if_already_run_today(time_t& start_date)
time_t begin_of_day;
utime_t now = ceph_clock_now(cct);
localtime_r(&start_date, &bdt);

if (cct->_conf->rgw_lc_debug_interval > 0) {
/* We're debugging, so say we can run */
return false;
}

bdt.tm_hour = 0;
bdt.tm_min = 0;
bdt.tm_sec = 0;
Expand Down Expand Up @@ -156,18 +162,32 @@ int RGWLC::bucket_lc_prepare(int index)
return 0;
}

bool RGWLC::obj_has_expired(double timediff, int days)
{
double cmp;

if (cct->_conf->rgw_lc_debug_interval <= 0) {
/* Normal case, run properly */
cmp = days*24*60*60;
} else {
/* We're in debug mode; Treat each rgw_lc_debug_interval seconds as a day */
cmp = days*cct->_conf->rgw_lc_debug_interval;
}

return (timediff >= cmp);
}

int RGWLC::bucket_lc_process(string& shard_id)
{
RGWLifecycleConfiguration config(cct);
RGWBucketInfo bucket_info;
map<string, bufferlist> bucket_attrs;
string prefix, delimiter, marker, next_marker, no_ns, end_marker, list_versions;
string next_marker, no_ns, list_versions;
bool is_truncated;
bool default_config = false;
int default_days = 0;
vector<RGWObjEnt> objs;
RGWObjectCtx obj_ctx(store);
map<string, bool> common_prefixes;
vector<std::string> result;
result = split(shard_id, ':');
string bucket_tenant = result[0];
Expand All @@ -188,12 +208,6 @@ int RGWLC::bucket_lc_process(string& shard_id)
RGWRados::Bucket target(store, bucket_info);
RGWRados::Bucket::List list_op(&target);

list_op.params.prefix = prefix;
list_op.params.delim = delimiter;
list_op.params.marker = marker;
list_op.params.end_marker = end_marker;
list_op.params.list_versions = false;

map<string, bufferlist>::iterator aiter = bucket_attrs.find(RGW_ATTR_LC);
if (aiter == bucket_attrs.end())
return 0;
Expand All @@ -211,7 +225,7 @@ int RGWLC::bucket_lc_process(string& shard_id)
if (prefix_iter->first.empty()) {
default_config = true;
default_days = prefix_iter->second;
continue;
break;
}
}

Expand All @@ -220,7 +234,7 @@ int RGWLC::bucket_lc_process(string& shard_id)

objs.clear();
list_op.params.marker = list_op.get_next_marker();
ret = list_op.list_objects(1000, &objs, &common_prefixes, &is_truncated);
ret = list_op.list_objects(1000, &objs, NULL, &is_truncated);
if (ret < 0) {
if (ret == -ENOENT)
return 0;
Expand Down Expand Up @@ -256,7 +270,7 @@ int RGWLC::bucket_lc_process(string& shard_id)
} else {
continue;
}
if (now - ceph::real_clock::to_time_t((*obj_iter).mtime) >= days*24*60*60) {
if (obj_has_expired(now - ceph::real_clock::to_time_t((*obj_iter).mtime), days)) {
RGWObjectCtx rctx(store);
rgw_obj obj(bucket_info.bucket, (*obj_iter).key.name);
RGWObjState *state;
Expand Down Expand Up @@ -286,7 +300,7 @@ int RGWLC::bucket_lc_process(string& shard_id)

objs.clear();
list_op.params.marker = list_op.get_next_marker();
ret = list_op.list_objects(1000, &objs, &common_prefixes, &is_truncated);
ret = list_op.list_objects(1000, &objs, NULL, &is_truncated);

if (ret < 0) {
if (ret == (-ENOENT))
Expand All @@ -300,7 +314,7 @@ int RGWLC::bucket_lc_process(string& shard_id)
utime_t now = ceph_clock_now(cct);

for (obj_iter = objs.begin(); obj_iter != objs.end(); obj_iter++) {
if (now - ceph::real_clock::to_time_t((*obj_iter).mtime) >= days*24*60*60) {
if (obj_has_expired(now - ceph::real_clock::to_time_t((*obj_iter).mtime), days)) {
RGWObjectCtx rctx(store);
rgw_obj obj(bucket_info.bucket, (*obj_iter).key.name);
RGWObjState *state;
Expand Down Expand Up @@ -511,17 +525,28 @@ bool RGWLC::LCWorker::should_work(utime_t& now)
struct tm bdt;
time_t tt = now.sec();
localtime_r(&tt, &bdt);
if ((bdt.tm_hour*60 + bdt.tm_min >= start_hour*60 + start_minute)||
(bdt.tm_hour*60 + bdt.tm_min <= end_hour*60 + end_minute)) {
return true;

if (cct->_conf->rgw_lc_debug_interval > 0) {
/* We're debugging, so say we can run */
return true;
} else if ((bdt.tm_hour*60 + bdt.tm_min >= start_hour*60 + start_minute) ||
(bdt.tm_hour*60 + bdt.tm_min <= end_hour*60 + end_minute)) {
return true;
} else {
return false;
return false;
}

}

int RGWLC::LCWorker::shedule_next_start_time(utime_t& now)
int RGWLC::LCWorker::schedule_next_start_time(utime_t &start, utime_t& now)
{
if (cct->_conf->rgw_lc_debug_interval > 0) {
int secs = start + cct->_conf->rgw_lc_debug_interval - now;
if (secs < 0)
secs = 0;
return (secs);
}

int start_hour;
int start_minute;
int end_hour;
Expand All @@ -536,6 +561,7 @@ int RGWLC::LCWorker::shedule_next_start_time(utime_t& now)
bdt.tm_min = start_minute;
bdt.tm_sec = 0;
nt = mktime(&bdt);

return (nt+24*60*60 - tt);
}

6 changes: 4 additions & 2 deletions src/rgw/rgw_lc.h
View file
Expand Up @@ -193,12 +193,11 @@ class RGWLC {
void *entry();
void stop();
bool should_work(utime_t& now);
int shedule_next_start_time(utime_t& now);
int schedule_next_start_time(utime_t& start, utime_t& now);
};

public:
LCWorker *worker;
public:
RGWLC() : cct(NULL), store(NULL), worker(NULL) {}
~RGWLC() {
stop_processor();
Expand All @@ -219,6 +218,9 @@ class RGWLC {
bool going_down();
void start_processor();
void stop_processor();

private:
bool obj_has_expired(double timediff, int days);
};


Expand Down

0 comments on commit 77853a4

Please sign in to comment.