Permalink
Browse files

Initial working version of memcached

git-svn-id: https://svn.opensource.yandex.net/xscript/trunk@734 b01ef89b-65f2-463d-9415-e8412542ae63
  • Loading branch information...
1 parent 849ce78 commit 71ad3d28d52f8096029b4a20702639e0913a1c37 bacek committed Feb 7, 2009
View
@@ -178,6 +178,9 @@ fi
AM_CONDITIONAL(HAVE_FILE_BLOCK, [test "f$ac_have_file" = "fyes"])
+AC_CHECK_HEADER([memcache.h], [ac_have_memcache="yes"], [ac_have_memcache="no"])
+AM_CONDITIONAL(HAVE_MEMCACHE, [test "f$ac_have_memcache" = "fyes"])
+
ac_have_dmalloc="no"
ac_dmalloc_found="no"
@@ -204,6 +207,7 @@ else
AC_MSG_RESULT([dmalloc not found. dmalloc check disabled])
fi
+
AX_CHECK_CPPUNIT
AC_ARG_ENABLE(coverage,
View
@@ -49,6 +49,14 @@ endif
xscript_validators_la_LIBADD = ../library/libxscript.la
xscript_validators_la_LDFLAGS = -module
+if HAVE_MEMCACHE
+pkglib_LTLIBRARIES += xscript-memcached.la
+xscript_memcached_la_SOURCES = doc_cache_memcached.cpp tag_key_memory.cpp
+xscript_memcached_la_CFLAGS = -g -O0
+xscript_memcached_la_LIBADD = ../library/libxscript.la
+xscript_memcached_la_LDFLAGS = -module -lmemcache
+endif
+
AM_CPPFLAGS = -I@top_srcdir@/include -I@builddir@/config -D_REENTRANT @yandex_platform_CFLAGS@
AM_CXXFLAGS = -W -Wall -fexceptions -frtti -ftemplate-depth-128 -finline -pthread
AM_LDFLAGS = -lboost_thread -lboost_filesystem @yandex_platform_LIBS@ @VERSION_INFO@
@@ -79,4 +87,9 @@ endif
test_LDADD = ../library/libxscript.la libtest.la
test_LDFLAGS = @CPPUNIT_LIBS@ -export-dynamic
+if HAVE_MEMCACHE
+test_SOURCES += memcached_test.cpp
+test_LDFLAGS += -lmemcache
+endif
+
endif
@@ -0,0 +1,172 @@
+#include <memcache.h>
+
+#include "xscript/doc_cache.h"
+#include "xscript/tag.h"
+#include "xscript/util.h"
+#include "xscript/xml_util.h"
+#include "tag_key_memory.h"
+
+#ifdef HAVE_DMALLOC_H
+#include <dmalloc.h>
+#endif
+
+#include <iostream>
+
+namespace xscript {
+
+/**
+ * TagKey for memcached. Hashes value from TagKeyMemory
+ */
+class TagKeyMemcached : public TagKeyMemory {
+public:
+ TagKeyMemcached(const Context *ctx, const TaggedBlock *block)
+ : TagKeyMemory(ctx, block)
+ {
+ value_ = HashUtils::hexMD5(value_.c_str(), value_.length());
+ }
+};
+
+/**
+ * Implementation of DocCacheStrategy using memcached.
+ */
+class DocCacheMemcached :
+ public Component<DocCacheMemcached>,
+ public DocCacheStrategy {
+public:
+ DocCacheMemcached();
+ virtual ~DocCacheMemcached();
+
+ virtual void init(const Config *config);
+
+ virtual time_t minimalCacheTime() const;
+
+ virtual std::auto_ptr<TagKey> createKey(const Context *ctx, const TaggedBlock *block) const;
+protected:
+ virtual bool loadDocImpl(const TagKey *key, Tag &tag, XmlDocHelper &doc);
+ virtual bool saveDocImpl(const TagKey *key, const Tag& tag, const XmlDocHelper &doc);
+
+private:
+ struct memcache *mc_;
+};
+
+DocCacheMemcached::DocCacheMemcached()
+{
+ mc_ = mc_new();
+ if (!mc_)
+ throw std::runtime_error("Unable to allocate new memcache object");
+
+ statBuilder_.setName("tagged-cache-memcached");
+ DocCache::instance()->addStrategy(this, "memcached");
+}
+
+DocCacheMemcached::~DocCacheMemcached()
+{
+ mc_free(mc_);
+}
+
+
+void
+DocCacheMemcached::init(const Config *config) {
+ log()->debug("initing Memcached");
+ std::vector<std::string> names;
+ config->subKeys(std::string("/xscript/tagged-cache-memcached/server"), names);
+ if (names.empty()) {
+ throw std::runtime_error("No memcached servers specified in config");
+ }
+
+ for (std::vector<std::string>::iterator i = names.begin(), end = names.end(); i != end; ++i) {
+ std::string server = config->as<std::string>(*i);
+ log()->debug("Adding %s", server.c_str());
+ mc_server_add4(mc_, server.c_str());
+ }
+}
+
+time_t
+DocCacheMemcached::minimalCacheTime() const {
+ return 5;
+}
+
+std::auto_ptr<TagKey>
+DocCacheMemcached::createKey(const Context *ctx, const TaggedBlock *block) const {
+ log()->debug("Creating key");
+ return std::auto_ptr<TagKey>(new TagKeyMemcached(ctx, block));
+}
+
+extern "C" int
+memcacheWriteFunc(void *ctx, const char *data, int len) {
+ std::string * str = static_cast<std::string*>(ctx);
+ str->append(data, len);
+ return len;
+}
+
+extern "C" int
+memcacheCloseFunc(void *ctx) {
+ (void) ctx;
+ return 0;
+}
+
+bool
+DocCacheMemcached::saveDocImpl(const TagKey *key, const Tag& tag, const XmlDocHelper &doc) {
+ log()->debug("saving doc");
+
+ std::string mc_key = key->asString();
+ std::string val;
+
+ // Adding Tag
+ val.append((char*)&tag.last_modified, sizeof(tag.last_modified));
+ val.append((char*)&tag.expire_time, sizeof(tag.expire_time));
+
+ xmlOutputBufferPtr buf = NULL;
+ buf = xmlOutputBufferCreateIO(&memcacheWriteFunc, &memcacheCloseFunc, &val, NULL);
+
+ xmlSaveFormatFileTo(buf, doc.get(), "UTF-8", 0);
+
+ char * mc_key2 = strdup(mc_key.c_str());
+ mc_set(mc_, mc_key2, mc_key.length(), val.c_str(), val.length(), tag.expire_time, 0);
+ free(mc_key2);
+ return true;
+}
+
+bool
+DocCacheMemcached::loadDocImpl(const TagKey *key, Tag &tag, XmlDocHelper &doc) {
+ log()->debug("loading doc");
+
+ std::string mc_key = key->asString();
+ char * mc_key2 = strdup(mc_key.c_str());
+ size_t vallen;
+ char * val = static_cast<char*>(mc_aget2(mc_, mc_key2, mc_key.length(), &vallen));
+ free(mc_key2);
+
+ if (!val) {
+ return false;
+ }
+
+ log()->debug("Loaded! %s", val);
+ try {
+ log()->debug("Parsing");
+
+ char * orig_val = val;
+
+ // Setting Tag
+ tag.last_modified = *((time_t*)(val));
+ val += sizeof(time_t);
+ tag.expire_time = *((time_t*)(val));
+ val += sizeof(time_t);
+
+ doc = XmlDocHelper(xmlParseMemory(val, vallen));
+ log()->debug("Parsed %p", doc.get());
+ free(orig_val);
+ XmlUtils::throwUnless(NULL != doc.get());
+ return true;
+ }
+ catch (const std::exception &e) {
+ log()->error("error while parsing doc from cache: %s", e.what());
+ return false;
+ }
+}
+
+namespace {
+ static DocCacheMemcached cache_;
+}
+
+};
No changes.
@@ -0,0 +1,10 @@
+<?xml version="1.0" ?>
+<?xml-stylesheet type="text/xsl" href="object.xsl"?>
+<page xmlns:x="http://www.yandex.ru/xscript" xmlns:xi="http://www.w3.org/2001/XInclude">
+ <xscript all-threaded="no"/>
+ <x:http>
+ <method>getHttp</method>
+ <param type="string">file://include.xml</param>
+ <param type="tag">5</param>
+ </x:http>
+</page>
View
@@ -0,0 +1,7 @@
+<?xml version="1.0" ?>
+<include-data>
+ <item>
+ <id>1</id>
+ <value>test</value>
+ </item>
+</include-data>
@@ -0,0 +1,26 @@
+<?xml version="1.0" ?>
+<xscript>
+ <variables>
+ <variable name="name">xscript</variable>
+ <variable name="pool-workers">5</variable>
+ </variables>
+ <endpoint>
+ <backlog>10</backlog>
+ <socket>/tmp/${name}.sock</socket>
+ </endpoint>
+ <pool-workers>${pool-workers}</pool-workers>
+ <logger-factory>
+ <logger>
+ <id>default</id>
+ <type>file</type>
+ <print-thread-id>yes</print-thread-id>
+ <level>debug</level>
+ <file>default.log</file>
+ </logger>
+ </logger-factory>
+ <modules>
+ <module id="tagged-cache-memcached">
+ <path>../standard/.libs/${name}-memcached.so</path>
+ </module>
+ </modules>
+</xscript>
@@ -0,0 +1,79 @@
+#include <iostream>
+
+#include <cppunit/TestFixture.h>
+#include <cppunit/extensions/HelperMacros.h>
+
+#include "xscript/config.h"
+#include "xscript/logger_factory.h"
+#include "internal/extension_list.h"
+#include "internal/loader.h"
+#include "xscript/doc_cache.h"
+#include "xscript/script.h"
+#include "xscript/request_data.h"
+#include "xscript/context.h"
+#include "xscript/tagged_block.h"
+
+using namespace xscript;
+
+
+class DocCacheMemcachedTest : public CppUnit::TestFixture {
+ CPPUNIT_TEST_SUITE(DocCacheMemcachedTest);
+
+ // This test works, but it crashes everything...
+ // CPPUNIT_TEST_EXCEPTION(testNoServers, std::exception);
+
+ CPPUNIT_TEST(testSaveLoad);
+
+
+ CPPUNIT_TEST_SUITE_END();
+private:
+
+ void testNoServers() {
+ try {
+ std::auto_ptr<Config> config = Config::create("memcached-noservers.conf");
+ config->startup();
+ }
+ catch(...) {
+ }
+ }
+
+ void testSaveLoad() {
+ std::auto_ptr<Config> config = Config::create("test.conf");
+ config->startup();
+
+ boost::shared_ptr<RequestData> data(new RequestData());
+ boost::shared_ptr<Script> script = Script::create("http-local-tagged.xml"); //cache_time==5
+ boost::shared_ptr<Context> ctx(new Context(script, data));
+ ContextStopper ctx_stopper(ctx);
+
+ XmlDocHelper doc(script->invoke(ctx));
+ CPPUNIT_ASSERT(NULL != doc.get());
+
+ const TaggedBlock* block = dynamic_cast<const TaggedBlock*>(script->block(0));
+ CPPUNIT_ASSERT(NULL != block);
+ CPPUNIT_ASSERT(block->tagged());
+
+ DocCache* tcache = DocCache::instance();
+
+ Tag tag_load;
+ XmlDocHelper doc_load;
+
+ time_t now = time(NULL);
+ Tag tag(true, now, now + 2);
+
+ // check save
+ CPPUNIT_ASSERT(tcache->saveDoc(ctx.get(), block, tag, doc));
+ CPPUNIT_ASSERT(NULL != doc.get());
+
+ // check load
+ CPPUNIT_ASSERT(tcache->loadDoc(ctx.get(), block, tag_load, doc_load));
+ CPPUNIT_ASSERT(NULL != doc_load.get());
+
+ sleep(3);
+ CPPUNIT_ASSERT(!tcache->loadDoc(ctx.get(), block, tag_load, doc_load));
+ }
+
+
+};
+
+CPPUNIT_TEST_SUITE_REGISTRATION( DocCacheMemcachedTest );
@@ -10,7 +10,7 @@ class TagKeyMemory : public TagKey {
TagKeyMemory(const Context *ctx, const TaggedBlock *block);
virtual const std::string& asString() const;
-private:
+protected:
std::string value_;
};
Oops, something went wrong.

0 comments on commit 71ad3d2

Please sign in to comment.