Skip to content
Newer
Older
100644 126 lines (96 sloc) 3.54 KB
24f6eee @chiyoung Schedule backfill task through Non-IO dispatcher.
chiyoung authored Jun 8, 2011
1 /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
2 #ifndef BACKFILL_HH
3 #define BACKFILL_HH 1
4
5 #include <assert.h>
6 #include <set>
7
8 #include "common.hh"
9 #include "stats.hh"
10 #include "dispatcher.hh"
11 #include "ep_engine.h"
12
790ff91 @chiyoung Pause backfill tasks if memory usage is too high.
chiyoung authored Jul 13, 2011
13 #define BACKFILL_MEM_THRESHOLD 0.9
24f6eee @chiyoung Schedule backfill task through Non-IO dispatcher.
chiyoung authored Jun 9, 2011
14
15 /**
16 * Dispatcher callback responsible for bulk backfilling tap queues
17 * from a KVStore.
18 *
19 * Note that this is only used if the KVStore reports that it has
20 * efficient vbucket ops.
21 */
954d02d @chiyoung MB-4402 Do not pass disk backfill dispatcher callback to kvstore.
chiyoung authored Nov 5, 2011
22 class BackfillDiskLoad : public DispatcherCallback {
24f6eee @chiyoung Schedule backfill task through Non-IO dispatcher.
chiyoung authored Jun 9, 2011
23 public:
24
25 BackfillDiskLoad(const std::string &n, EventuallyPersistentEngine* e,
26 TapConnMap &tcm, KVStore *s, uint16_t vbid, const void *token)
56ba211 @chiyoung VBucket version check is not necessary for vb backfill from disk.
chiyoung authored Nov 5, 2011
27 : name(n), engine(e), connMap(tcm), store(s), vbucket(vbid), validityToken(token) { }
24f6eee @chiyoung Schedule backfill task through Non-IO dispatcher.
chiyoung authored Jun 9, 2011
28
29 void callback(GetValue &gv);
30
31 bool callback(Dispatcher &, TaskId);
32
33 std::string description();
34
35 private:
36 const std::string name;
37 EventuallyPersistentEngine *engine;
38 TapConnMap &connMap;
39 KVStore *store;
40 uint16_t vbucket;
41 const void *validityToken;
42 };
43
44 /**
45 * VBucketVisitor to backfill a TapProducer. This visitor basically performs backfill from memory
46 * for only resident items if it needs to schedule a separate disk backfill task because of
47 * low resident ratio.
48 */
49 class BackFillVisitor : public VBucketVisitor {
50 public:
51 BackFillVisitor(EventuallyPersistentEngine *e, TapProducer *tc,
52 const void *token, const VBucketFilter &backfillVBfilter):
63fd705 @chiyoung Allow TAP backfill to pass its vbucket filter to VBucketVisitor
chiyoung authored Sep 1, 2011
53 VBucketVisitor(backfillVBfilter), engine(e), name(tc->getName()),
24f6eee @chiyoung Schedule backfill task through Non-IO dispatcher.
chiyoung authored Jun 9, 2011
54 queue(new std::list<queued_item>),
40e5927 @chiyoung Merge branch 'branch-172' into branch-20
chiyoung authored Sep 22, 2011
55 found(),
1471282 @trondn Refactor: Make TAP configuration variables non-static
trondn authored Jul 27, 2011
56 validityToken(token), valid(true),
24f6eee @chiyoung Schedule backfill task through Non-IO dispatcher.
chiyoung authored Jun 9, 2011
57 efficientVBDump(e->epstore->getStorageProperties().hasEfficientVBDump()),
58 residentRatioBelowThreshold(false) {
1471282 @trondn Refactor: Make TAP configuration variables non-static
trondn authored Jul 27, 2011
59
60 found.reserve(e->getTapConfig().getBackfillBacklogLimit());
24f6eee @chiyoung Schedule backfill task through Non-IO dispatcher.
chiyoung authored Jun 9, 2011
61 }
62
63 virtual ~BackFillVisitor() {
64 delete queue;
65 }
66
4a73f0e @trondn Use reference passing instead of copy constructor
trondn authored Dec 19, 2011
67 bool visitBucket(RCPtr<VBucket> &vb);
24f6eee @chiyoung Schedule backfill task through Non-IO dispatcher.
chiyoung authored Jun 9, 2011
68
69 void visit(StoredValue *v);
70
71 bool shouldContinue() {
72 return checkValidity();
73 }
74
75 void apply(void);
76
77 void complete(void);
78
79 private:
80
81 void setEvents();
82
83 bool pauseVisitor();
84
85 bool checkValidity();
86
87 EventuallyPersistentEngine *engine;
88 const std::string name;
89 std::list<queued_item> *queue;
90 std::vector<std::pair<uint16_t, queued_item> > found;
91 std::vector<uint16_t> vbuckets;
92 const void *validityToken;
93 bool valid;
94 bool efficientVBDump;
95 bool residentRatioBelowThreshold;
96 };
97
98 /**
99 * Backfill task scheduled by non-IO dispatcher. Each backfill task performs backfill from
100 * memory or disk depending on the resident ratio. Each backfill task can backfill more than one
101 * vbucket, but will snooze for 1 sec if the current backfill backlog for the corresponding TAP
102 * producer is greater than the threshold (5000 by default).
103 */
104 class BackfillTask : public DispatcherCallback {
105 public:
106
107 BackfillTask(EventuallyPersistentEngine *e, TapProducer *tc,
108 EventuallyPersistentStore *s, const void *tok,
109 const VBucketFilter &backfillVBFilter):
110 bfv(new BackFillVisitor(e, tc, tok, backfillVBFilter)), engine(e), epstore(s) {}
111
112 virtual ~BackfillTask() {}
113
114 bool callback(Dispatcher &d, TaskId t);
115
116 std::string description() {
117 return std::string("Backfilling items from memory and disk.");
118 }
119
120 shared_ptr<BackFillVisitor> bfv;
121 EventuallyPersistentEngine *engine;
122 EventuallyPersistentStore *epstore;
123 };
124
125 #endif /* BACKFILL_HH */
Something went wrong with that request. Please try again.