LifeCycle feature

As same as amazon S3 interface,"PUT Bucket lifecycle" and
"DELETE Bucket lifecycle" have been implemented,
"GET Bucket lifecycle" not realized yet as S3cmd has not
realize it also.
The feature`s main point is to remove expire file per day.
As ceph does not have a tier concept, so files transfer
from hot layer to cold layer is also not supported.

lifecycle config file format:
<LifecycleConfiguration>
    <Rule>
        <ID>sample-rule</ID>
        <Prefix></Prefix>
        <Status>enable</Status>
        <Expiration>
           <Days>1</Days>
        </Expiration>
    </Rule>
</LifecycleConfiguration>

Signed-off-by: Ji Chen <insomnia@139.com>

src/common/config_opts.h

@@ -1031,6 +1031,8 @@ OPTION(rgw_bucket_index_max_aio, OPT_U32, 8)
  */
 OPTION(rgw_enable_quota_threads, OPT_BOOL, true)
 OPTION(rgw_enable_gc_threads, OPT_BOOL, true)
+OPTION(rgw_enable_lc_threads, OPT_BOOL, true)
+
 
 OPTION(rgw_data, OPT_STR, "/var/lib/ceph/radosgw/$cluster-$id")
 OPTION(rgw_enable_apis, OPT_STR, "s3, swift, swift_auth, admin")
@@ -1041,6 +1043,9 @@ OPTION(rgw_host, OPT_STR, "")  // host for radosgw, can be an IP, default is 0.0
 OPTION(rgw_port, OPT_STR, "")  // port to listen, format as "8080" "5000", if not specified, rgw will not run external fcgi
 OPTION(rgw_dns_name, OPT_STR, "")
 OPTION(rgw_content_length_compat, OPT_BOOL, false) // Check both HTTP_CONTENT_LENGTH and CONTENT_LENGTH in fcgi env
+OPTION(rgw_lifecycle_enabled, OPT_BOOL, true) //rgw lifecycle enabled
+OPTION(rgw_lifecycle_thread, OPT_INT, 1) //start lifecycle thread number per radosgw
+OPTION(rgw_lifecycle_work_time, OPT_STR, "00:00-06:00") //job process lc  at 00:00-06:00s
 OPTION(rgw_script_uri, OPT_STR, "") // alternative value for SCRIPT_URI if not set in request
 OPTION(rgw_request_uri, OPT_STR,  "") // alternative value for REQUEST_URI if not set in request
 OPTION(rgw_swift_url, OPT_STR, "")             // the swift url, being published by the internal swift auth

src/rgw/Makefile.am

@@ -16,6 +16,8 @@ librgw_la_SOURCES =  \
 	rgw/rgw_acl.cc \
 	rgw/rgw_acl_s3.cc \
 	rgw/rgw_acl_swift.cc \
+	rgw/rgw_lc.cc \
+	rgw/rgw_lc_s3.cc \
 	rgw/rgw_client_io.cc \
 	rgw/rgw_fcgi.cc \
 	rgw/rgw_xml.cc \
@@ -127,6 +129,8 @@ noinst_HEADERS += \
 	rgw/rgw_acl.h \
 	rgw/rgw_acl_s3.h \
 	rgw/rgw_acl_swift.h \
+	rgw/rgw_lc.h \
+	rgw/rgw_lc_s3.h \
 	rgw/rgw_client_io.h \
 	rgw/rgw_fcgi.h \
 	rgw/rgw_xml.h \

src/rgw/rgw_admin.cc

@@ -95,6 +95,8 @@ void _usage()
   cout << "  gc list                    dump expired garbage collection objects (specify\n";
   cout << "                             --include-all to list all entries, including unexpired)\n";
   cout << "  gc process                 manually process garbage\n";
+  cout << "  lc list                    list all bucket lifecycle progress\n";
+  cout << "  lc process                 manually process lifecycle\n";
   cout << "  metadata get               get metadata info\n";
   cout << "  metadata put               put metadata info\n";
   cout << "  metadata rm                remove metadata info\n";
@@ -238,6 +240,8 @@ enum {
   OPT_QUOTA_DISABLE,
   OPT_GC_LIST,
   OPT_GC_PROCESS,
+  OPT_LC_LIST,
+  OPT_LC_PROCESS,
   OPT_ORPHANS_FIND,
   OPT_ORPHANS_FINISH,
   OPT_REGION_GET,
@@ -284,6 +288,7 @@ static int get_cmd(const char *cmd, const char *prev_cmd, bool *need_more)
       strcmp(cmd, "gc") == 0 || 
       strcmp(cmd, "key") == 0 ||
       strcmp(cmd, "log") == 0 ||
+      strcmp(cmd, "lc") == 0 ||
       strcmp(cmd, "mdlog") == 0 ||
       strcmp(cmd, "metadata") == 0 ||
       strcmp(cmd, "object") == 0 ||
@@ -454,6 +459,11 @@ static int get_cmd(const char *cmd, const char *prev_cmd, bool *need_more)
       return OPT_GC_LIST;
     if (strcmp(cmd, "process") == 0)
       return OPT_GC_PROCESS;
+  } else if (strcmp(prev_cmd, "lc") == 0) {
+    if (strcmp(cmd, "list") == 0)
+      return OPT_LC_LIST;
+    if (strcmp(cmd, "process") == 0)
+      return OPT_LC_PROCESS;
   } else if (strcmp(prev_cmd, "orphans") == 0) {
     if (strcmp(cmd, "find") == 0)
       return OPT_ORPHANS_FIND;
@@ -1437,7 +1447,7 @@ int main(int argc, char **argv)
   if (raw_storage_op) {
     store = RGWStoreManager::get_raw_storage(g_ceph_context);
   } else {
-    store = RGWStoreManager::get_storage(g_ceph_context, false, false);
+    store = RGWStoreManager::get_storage(g_ceph_context, false, false, false);
   }
   if (!store) {
     cerr << "couldn't init storage provider" << std::endl;
@@ -2615,6 +2625,44 @@ int main(int argc, char **argv)
     }
   }
 
+  if (opt_cmd == OPT_LC_LIST) {
+
+
+      const char* LC_STATUS[] = {
+      "UNDEFED",
+      "PROCESSING",
+      "WAITING",
+      "COMPLETE"
+      };
+      formatter->open_array_section("life cycle progress");
+
+      do {
+        map<string, int> bucket_lc_map;
+        int ret = store->list_lc_progress(bucket_lc_map);
+        if (ret < 0) {
+    cerr << "ERROR: failed to list objs: " << cpp_strerror(-ret) << std::endl;
+    return 1;
+        }
+        map<string, int>::iterator iter;
+        for (iter = bucket_lc_map.begin(); iter != bucket_lc_map.end(); ++iter) {
+    formatter->open_object_section("bucket_lc_info");
+    formatter->dump_string("bucket", iter->first);
+    string lc_status = LC_STATUS[iter->second];
+    formatter->dump_string("status", lc_status);
+    formatter->close_section(); // objs          
+    formatter->flush(cout);
+        }
+      } while (0);
+    }
+
+    if (opt_cmd == OPT_LC_PROCESS) {
+      int ret = store->process_lc();
+      if (ret < 0) {
+        cerr << "ERROR: lc processing returned error: " << cpp_strerror(-ret) << std::endl;
+        return 1;
+      }
+    }
+
   if (opt_cmd == OPT_ORPHANS_FIND) {
     RGWOrphanSearch search(store, max_concurrent_ios, orphan_stale_secs);
 

src/rgw/rgw_common.cc

@@ -603,6 +603,7 @@ int RGWHTTPArgs::parse()
       }
 
       if ((name.compare("acl") == 0) ||
+          (name.compare("lifecycle") == 0) ||
           (name.compare("cors") == 0) ||
           (name.compare("location") == 0) ||
           (name.compare("logging") == 0) ||

src/rgw/rgw_common.h

@@ -55,6 +55,7 @@ using ceph::crypto::MD5;
 #define RGW_SYS_PARAM_PREFIX "rgwx-"
 
 #define RGW_ATTR_ACL		RGW_ATTR_PREFIX "acl"
+#define RGW_ATTR_LC		    RGW_ATTR_PREFIX "lc"
 #define RGW_ATTR_CORS		RGW_ATTR_PREFIX "cors"
 #define RGW_ATTR_ETAG    	RGW_ATTR_PREFIX "etag"
 #define RGW_ATTR_BUCKETS	RGW_ATTR_PREFIX "buckets"