Skip to content

Commit

Permalink
Made checkfiles.pl faster and refactored SearchStore
Browse files Browse the repository at this point in the history
  • Loading branch information
Caleb James DeLisle committed Mar 9, 2013
1 parent 258cfc1 commit 2c86e4a
Show file tree
Hide file tree
Showing 11 changed files with 234 additions and 571 deletions.
28 changes: 1 addition & 27 deletions cmake/modules/CheckCodestyle.cmake
View file
Expand Up @@ -14,34 +14,8 @@ find_program(perl perl)
if(NOT perl)
message("Couldn't find perl, skipping codestyle check.")
else()
string(REPLACE "," ";" skipDirs "${SKIP}")
set(filesToCheck "")
file(GLOB_RECURSE files "${CSD}/*.[ch]")
foreach(f ${files})
set(skip FALSE)
foreach(dir ${skipDirs})
if (${f} MATCHES "${CSD}/${dir}/.*")
set(skip TRUE)
break()
endif()
endforeach()
# ignore all *build* subdirectories
if (${f} MATCHES "${CSD}/[^/]*build[^/]*/.*")
set(skip TRUE)
endif()
if (skip OR IS_DIRECTORY ${f})
# skip
elseif(${f} MATCHES "${CSD}/.*")
#message("checking ${f}")
list(APPEND filesToCheck "${f}\n")
endif()
endforeach()
string(REPLACE ";" "" fileList "${filesToCheck}")
file(REMOVE ${CMAKE_BINARY_DIR}/files_to_check.txt)
file(WRITE ${CMAKE_BINARY_DIR}/files_to_check.txt ${fileList})

execute_process(COMMAND ${perl} ${CSD}/scripts/checkfiles.pl
INPUT_FILE ${CMAKE_BINARY_DIR}/files_to_check.txt
WORKING_DIRECTORY ${CSD}
OUTPUT_VARIABLE out
)
if (NOT "${out}" STREQUAL "")
Expand Down
6 changes: 4 additions & 2 deletions dht/dhtcore/Janitor.c
View file
Expand Up @@ -114,7 +114,8 @@ static void maintanenceCycle(void* vcontext)
RouterModule_beginSearch(targetAddr.ip6.bytes,
searchStepCallback,
janitor,
janitor->routerModule);
janitor->routerModule,
janitor->allocator);
return;
}

Expand All @@ -140,7 +141,8 @@ static void maintanenceCycle(void* vcontext)
RouterModule_beginSearch(targetAddr.ip6.bytes,
searchStepCallback,
janitor,
janitor->routerModule);
janitor->routerModule,
janitor->allocator);
janitor->timeOfNextGlobalMaintainence += janitor->globalMaintainenceMilliseconds;
}
}
Expand Down
94 changes: 39 additions & 55 deletions dht/dhtcore/RouterModule.c
View file
Expand Up @@ -248,7 +248,7 @@ struct RouterModule* RouterModule_register(struct DHTModuleRegistry* registry,

out->gmrtRoller = AverageRoller_new(GMRT_SECONDS, eventBase, allocator);
AverageRoller_update(out->gmrtRoller, GMRT_INITAL_MILLISECONDS);
out->searchStore = SearchStore_new(allocator, out->gmrtRoller, eventBase, logger);
out->searchStore = SearchStore_new(allocator, out->gmrtRoller, eventBase, rand, logger);
out->nodeStore = NodeStore_new(&out->address, NODE_STORE_SIZE, allocator, logger, admin);
out->registry = registry;
out->eventBase = eventBase;
Expand Down Expand Up @@ -489,6 +489,8 @@ struct SearchCallbackContext

struct SearchStore_Node* lastNodeCalled;

struct Allocator* alloc;

/** The number of milliseconds before timing out a request. */
uint64_t timeoutMilliseconds;
};
Expand All @@ -502,40 +504,36 @@ struct SearchCallbackContext
static void searchStep(struct SearchCallbackContext* scc)
{
struct RouterModule* module = scc->routerModule;
struct Allocator* searchAllocator = SearchStore_getAllocator(scc->search);

struct SearchStore_Node* nextSearchNode =
SearchStore_getNextNode(scc->search, searchAllocator);
struct SearchStore_Node* nextSearchNode = SearchStore_getNextNode(scc->search);

// If the number of requests sent has exceeded the max search requests, let's stop there.
if (scc->totalRequests >= MAX_REQUESTS_PER_SEARCH || nextSearchNode == NULL) {
if (scc->resultCallback != NULL) {
scc->resultCallback(scc->resultCallbackContext, NULL);
}
SearchStore_freeSearch(scc->search);
Allocator_free(scc->alloc);
return;
}

// Get the node from the nodestore because there might be a much better path to the same node.
struct Node* n = NodeStore_getBest(nextSearchNode->address, scc->routerModule->nodeStore);
if (n && !Bits_memcmp(n->address.ip6.bytes, nextSearchNode->address->ip6.bytes, 16)) {
struct Node* n = NodeStore_getBest(&nextSearchNode->address, scc->routerModule->nodeStore);
if (n && !Bits_memcmp(n->address.ip6.bytes, nextSearchNode->address.ip6.bytes, 16)) {
uint64_t nlabel = n->address.path;
uint64_t nsn = nextSearchNode->address->path;
uint64_t nsn = nextSearchNode->address.path;
if (nlabel < nsn) {
nextSearchNode->address = &n->address;
nextSearchNode->address.path = n->address.path;
}
}

sendRequest(nextSearchNode->address,
sendRequest(&nextSearchNode->address,
scc->requestType,
SearchStore_tidForNode(nextSearchNode, searchAllocator),
nextSearchNode->txid,
&scc->targetAddress,
scc->targetKey,
scc->routerModule->registry);

scc->totalRequests++;
scc->lastNodeCalled = nextSearchNode;
SearchStore_requestSent(nextSearchNode, module->searchStore);
scc->timeoutMilliseconds = tryNextNodeAfter(module);
Timeout_resetTimeout(scc->timeout, scc->timeoutMilliseconds);
}
Expand All @@ -558,7 +556,7 @@ static uint32_t reachAfterTimeout(const uint32_t oldReach)
static void searchRequestTimeout(void* vcontext)
{
struct SearchCallbackContext* scc = (struct SearchCallbackContext*) vcontext;
struct Node* n = NodeStore_getNodeByNetworkAddr(scc->lastNodeCalled->address->path,
struct Node* n = NodeStore_getNodeByNetworkAddr(scc->lastNodeCalled->address.path,
scc->routerModule->nodeStore);

// Search timeout -> set to 0 reach.
Expand Down Expand Up @@ -774,8 +772,7 @@ static inline int handleReply(struct DHTMessage* message, struct RouterModule* m
return -1;
}

struct SearchStore_Node* parent =
SearchStore_getNode(tid, module->searchStore, message->allocator);
struct SearchStore_Node* parent = SearchStore_getNode(tid, module->searchStore);

if (parent == NULL) {
// Couldn't find the node, perhaps we were sent a malformed packet.
Expand All @@ -785,10 +782,10 @@ static inline int handleReply(struct DHTMessage* message, struct RouterModule* m

// If the search has already replaced the node's location or it has already finished
// and another search is taking place in the same slot, drop this reply because it is late.
if (!Address_isSameIp(parent->address, message->address)) {
if (!Address_isSameIp(&parent->address, message->address)) {
#ifdef Log_DEBUG
uint8_t expectedAddr[60];
Address_print(expectedAddr, parent->address);
Address_print(expectedAddr, &parent->address);
uint8_t receivedAddr[60];
Address_print(receivedAddr, message->address);
Log_debug(module->logger,
Expand All @@ -801,10 +798,10 @@ static inline int handleReply(struct DHTMessage* message, struct RouterModule* m
return -1;
}

responseFromNode(node, SearchStore_replyReceived(parent, module->searchStore), module);
responseFromNode(node, SearchStore_replyReceived(parent), module);

struct SearchStore_Search* search = SearchStore_getSearchForNode(parent, module->searchStore);
struct SearchCallbackContext* scc = SearchStore_getContext(search);
struct SearchStore_Search* search = parent->search;
struct SearchCallbackContext* scc = search->callbackContext;

if (!nodes) {
// They didn't have anything to give us.
Expand All @@ -829,9 +826,8 @@ static inline int handleReply(struct DHTMessage* message, struct RouterModule* m
// If this node has sent us any entries which are further from the target than it is,
// garbage the whole response.
const uint32_t targetPrefix = Address_getPrefix(&scc->targetAddress);
const uint32_t parentDistance = Address_getPrefix(parent->address) ^ targetPrefix;
const uint32_t parentDistance = Address_getPrefix(&parent->address) ^ targetPrefix;

uint64_t evictTime = evictUnrepliedIfOlderThan(module);
for (uint32_t i = 0; i < nodes->len; i += Address_SERIALIZED_SIZE) {
if (isDuplicateEntry(nodes, i)) {
continue;
Expand Down Expand Up @@ -885,25 +881,21 @@ static inline int handleReply(struct DHTMessage* message, struct RouterModule* m
NodeStore_addNode(module->nodeStore, &addr, 0, version);

if ((newNodePrefix ^ targetPrefix) >= parentDistance
&& xorCompare(&scc->targetAddress, &addr, parent->address) >= 0)
&& xorCompare(&scc->targetAddress, &addr, &parent->address) >= 0)
{
// Too much noise.
//Log_debug(module->logger, "Answer was further from the target than us.\n");
continue;
}

struct Node* n = NodeStore_getBest(&addr, module->nodeStore);
SearchStore_addNodeToSearch(parent, (n) ? &n->address : &addr, evictTime, search);
SearchStore_addNodeToSearch((n) ? &n->address : &addr, search);
}

// Ask the callback if we should continue.
if (scc->resultCallback == NULL
|| scc->resultCallback(scc->resultCallbackContext, message) == false)
{
searchStep(scc);
} else {
SearchStore_freeSearch(search);
if (scc->resultCallback == NULL) {
scc->resultCallback(scc->resultCallbackContext, message);
}
searchStep(scc);
return 0;
}

Expand Down Expand Up @@ -1022,17 +1014,19 @@ static int handleOutgoing(struct DHTMessage* message, void* vcontext)

/** See: RouterModule.h */
struct RouterModule_Search* RouterModule_beginSearch(
const uint8_t searchTarget[Address_SEARCH_TARGET_SIZE],
uint8_t searchTarget[16],
bool (* const callback)(void* callbackContext, struct DHTMessage* result),
void* callbackContext,
struct RouterModule* module)
struct RouterModule* module,
struct Allocator* alloc)
{
struct SearchStore_Search* search = SearchStore_newSearch(searchTarget, module->searchStore);
struct Allocator* searchAllocator = Allocator_child(alloc);
struct SearchStore_Search* search =
SearchStore_newSearch(searchTarget, module->searchStore, searchAllocator);
if (!search) {
Log_debug(module->logger, "Can't run search because SearchStore is full.");
return NULL;
}
struct Allocator* searchAllocator = SearchStore_getAllocator(search);

struct Address targetAddr;
Bits_memcpyConst(targetAddr.ip6.bytes, searchTarget, Address_SEARCH_TARGET_SIZE);
Expand All @@ -1048,35 +1042,30 @@ struct RouterModule_Search* RouterModule_beginSearch(

if (nodes->size == 0) {
Log_debug(module->logger, "Can't find any nodes to begin search.");
SearchStore_freeSearch(search);
Allocator_free(search->alloc);
return NULL;
}

for (uint32_t i = 0; i < nodes->size; i++) {
SearchStore_addNodeToSearch(NULL,
&nodes->nodes[i]->address,
evictUnrepliedIfOlderThan(module),
search);
SearchStore_addNodeToSearch(&nodes->nodes[i]->address, search);
}

struct SearchStore_Node* firstSearchNode = SearchStore_getNextNode(search, searchAllocator);
struct SearchStore_Node* firstSearchNode = SearchStore_getNextNode(search);

#ifdef Log_DEBUG
uint8_t addr[60];
Address_print(addr, firstSearchNode->address);
Address_print(addr, &firstSearchNode->address);
Log_debug(module->logger, "Search %s\n", addr);
#endif

// Send out the request.
sendRequest(firstSearchNode->address,
sendRequest(&firstSearchNode->address,
CJDHTConstants_QUERY_FN,
SearchStore_tidForNode(firstSearchNode, searchAllocator),
firstSearchNode->txid,
&targetAddr,
CJDHTConstants_TARGET,
module->registry);

SearchStore_requestSent(firstSearchNode, module->searchStore);

struct SearchCallbackContext* scc =
Allocator_malloc(searchAllocator, sizeof(struct SearchCallbackContext));

Expand All @@ -1097,12 +1086,13 @@ struct RouterModule_Search* RouterModule_beginSearch(
.requestType = CJDHTConstants_QUERY_FN,
.targetKey = CJDHTConstants_TARGET,
.lastNodeCalled = firstSearchNode,
.timeoutMilliseconds = timeoutMilliseconds
.timeoutMilliseconds = timeoutMilliseconds,
.alloc = searchAllocator
};
Bits_memcpyConst(scc, &sccLocal, sizeof(struct SearchCallbackContext));
Bits_memcpyConst(&scc->targetAddress, &targetAddr, sizeof(struct Address));

SearchStore_setContext(scc, search);
search->callbackContext = scc;

struct RouterModule_Search* out =
Allocator_malloc(searchAllocator, sizeof(struct RouterModule_Search));
Expand All @@ -1111,12 +1101,6 @@ struct RouterModule_Search* RouterModule_beginSearch(
return out;
}

/** See: RouterModule.h */
void RouterModule_cancelSearch(struct RouterModule_Search* toCancel)
{
SearchStore_freeSearch(toCancel->search);
}

int RouterModule_brokenPath(const uint64_t path, struct RouterModule* module)
{
return NodeStore_brokenPath(path, module->nodeStore);
Expand Down
21 changes: 8 additions & 13 deletions dht/dhtcore/RouterModule.h
View file
Expand Up @@ -72,22 +72,17 @@ struct RouterModule* RouterModule_register(struct DHTModuleRegistry* registry,
* if it returns true then the search is assumed to be complete.
* @param callbackContext a pointer which will be passed back to the callback when it is called.
* @param module the router module which should perform the search.
* @param searchAllocator a temporary allocator to use for allocating the search data.
* freeing this allocator will cancel the search.
* @return a search if all goes well, NULL if the search could not be completed because there are
* no nodes closer to the destination than us or if there is not enough empty search slots.
*/
struct RouterModule_Search*
RouterModule_beginSearch(const uint8_t target[Address_SEARCH_TARGET_SIZE],
bool (* const callback)(void* callbackContext,
struct DHTMessage* result),
void* callbackContext,
struct RouterModule* module);

/**
* Cancel a search before it is complete.
*
* @param toCancel the result of calling RouterModule_beginSearch() for this search.
*/
void RouterModule_cancelSearch(struct RouterModule_Search* toCancel);
struct RouterModule_Search* RouterModule_beginSearch(
uint8_t searchTarget[16],
bool (* const callback)(void* callbackContext, struct DHTMessage* result),
void* callbackContext,
struct RouterModule* module,
struct Allocator* searchAllocator);

/**
* Manually add a node to the routing table.
Expand Down

0 comments on commit 2c86e4a

Please sign in to comment.