-
Notifications
You must be signed in to change notification settings - Fork 1
/
mmap.cpp
138 lines (115 loc) · 2.93 KB
/
mmap.cpp
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>
#include <sys/mman.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#include <iostream>
#include <fstream>
#include <cstring>
#include "glog/logging.h"
#include "baselib/text.h"
#include "mmap.h"
using namespace std;
static void deleter(const Slice& key, void* value) {
if(value) {
free(value);
value = NULL;
}
}
ReadonlyMMap::ReadonlyMMap() : _fd(-1), _ptr(NULL), _fsize(0), _cache(NULL) { }
bool ReadonlyMMap::init(const string& index_fn, const string& data_fn) {
_index_fn = index_fn;
_data_fn = data_fn;
_fd = open(_data_fn.c_str(), O_RDWR);
if(_fd == -1) {
LOG(INFO) << "open file " << _data_fn << " fail";
return false;
}
struct stat sb;
fstat(_fd, &sb);
_fsize = sb.st_size;
_ptr = (char* )mmap(NULL, sb.st_size, PROT_READ, MAP_SHARED, _fd, 0);
if (_ptr == MAP_FAILED) {
LOG(INFO) << "mmap data file fail";
return false;
}
LOG(INFO) << "begin to load index file";
FILE* fin = fopen(_index_fn.c_str(), "r");
if(fin == NULL) {
LOG(INFO) << "open index file " << _index_fn << " fail";
return false;
}
const int LINE_SIZE = 1024;
char line[LINE_SIZE];
while(fgets(line, LINE_SIZE, fin) != NULL) {
// LOG(INFO) << "read line " << line;
std::vector<std::string> i = StringSplitter::split(line, "\t");
if(i.size() != 2) {
LOG(INFO) << " read bad line " << line;
continue;
}
size_t h = _hash(i[0]);
off_t offset = atol(i[1].c_str());
_index[h] = offset;
// LOG(INFO) << "init_mmap hash value " << h << " offset " << offset;
}
fclose(fin);
// 加载cache
const int cache_size = 1000000;
_cache = NewLRUCache(cache_size);
if(_cache == NULL) {
LOG(INFO) << "new lrucache fail";
return false;
}
return true;
}
ReadonlyMMap::~ReadonlyMMap() {
if(_ptr)
munmap(_ptr, _fsize);
if(_fd != -1)
close(_fd);
}
bool ReadonlyMMap::seek(const string& k, string& v) {
// 命中cache
Cache::Handle* handle = _cache->Lookup(k);
if(handle != NULL) {
LOG(INFO) << "hit cache for key " << k;
v.clear();
v = (char*)_cache->Value(handle);
_cache->Release(handle);
return true;
}
size_t h = _hash(k);
if(_index.find(h) == _index.end()) {
return false;
}
off_t offset = _index[h];
LOG(INFO) << "userid " << k << " hash " << h << " offset " << offset << endl;
off_t s = offset;
while(true) {
if(offset <= _fsize && _ptr[offset] != '\n') {
offset++;
}
else {
break;
}
}
v.clear();
v.append(&_ptr[s], offset - s + 1);
LOG(INFO) << "seek key " << k << " value " << v << " init offset " << s << " after offset " << offset;
// 将数据添加到cache中
void* value = malloc(sizeof(char) * v.size());
if(value != NULL) {
memcpy(value, v.c_str(), v.size());
Cache::Handle* handle = _cache->Insert(k.c_str(), value, 1, deleter);
_cache->Release(handle);
LOG(INFO) << "insert cache for k " << k;
}
else {
LOG(INFO) << "malloc cache memory fail";
}
return true;
}