Skip to content
Newer
Older
100644 330 lines (287 sloc) 12.6 KB
7516943 added MIT license
sammy authored Aug 25, 2010
1 /*
183680d First code checkin
sammy authored Aug 23, 2010
2 * This file implements saving alsosql datastructures to rdb files
7516943 added MIT license
sammy authored Aug 25, 2010
3 *
4
5 MIT License
6
32efb18 CREATE TABLE AS now supports all redis READ operations (e.g. SELECT,L…
sammy authored Aug 27, 2010
7 Copyright (c) 2010 Russell Sullivan <jaksprats AT gmail DOT com>
8 ALL RIGHTS RESERVED
7516943 added MIT license
sammy authored Aug 25, 2010
9
10 Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
11
12 The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
13
14 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
15
183680d First code checkin
sammy authored Aug 24, 2010
16 */
17 #include <stdio.h>
18 #include <stdlib.h>
19 #include <string.h>
20
21 #include "redis.h"
22 #include "dict.h"
23 #include "index.h"
24 #include "bt.h"
25 #include "common.h"
ef35f30 Tbl[] refactored
sammy authored Oct 1, 2010
26 #include "alsosql.h"
183680d First code checkin
sammy authored Aug 24, 2010
27 #include "rdb_alsosql.h"
28
29 // FROM redis.c
30 #define RL4 redisLog(4,
036c922 ruby client showed some bugs
sammy authored Sep 30, 2010
31 extern struct redisServer server;
183680d First code checkin
sammy authored Aug 24, 2010
32
7f422d7 additional dimension to Tbl[]
sammy authored Oct 1, 2010
33 extern int Num_tbls [MAX_NUM_TABLES];
34 extern r_tbl_t Tbl[MAX_NUM_DB][MAX_NUM_TABLES];
183680d First code checkin
sammy authored Aug 24, 2010
35
75ade31 Index[] now respects dbid .... namespaces in SQL work
sammy authored Oct 1, 2010
36 extern int Num_indx[MAX_NUM_DB];
37 extern r_ind_t Index [MAX_NUM_DB][MAX_NUM_INDICES];
183680d First code checkin
sammy authored Aug 24, 2010
38
75ade31 Index[] now respects dbid .... namespaces in SQL work
sammy authored Oct 2, 2010
39 extern char *COLON;
183680d First code checkin
sammy authored Aug 24, 2010
40
41 unsigned char VIRTUAL_INDEX_TYPE = 255;
42
db3f032 digging the rabbit hole very deep - the non relational index .... sti…
sammy authored Oct 19, 2010
43 int rdbSaveNRL(FILE *fp, robj *o) {
44 listNode *ln;
45 d_l_t *nrlind = o->ptr;
46 list *nrltoks = nrlind->l1;
47
48 int imatch = nrlind->num;
49 if (rdbSaveLen(fp, imatch) == -1) return -1;
50 robj *iname = Index[server.dbid][imatch].obj;
51 if (rdbSaveStringObject(fp, iname) == -1) return -1;
52 int tmatch = Index[server.dbid][imatch].table;
53 if (rdbSaveLen(fp, tmatch) == -1) return -1;
54
55 if (rdbSaveLen(fp, listLength(nrltoks)) == -1) return -1;
56 listIter *li = listGetIterator(nrltoks, AL_START_HEAD);
57 while((ln = listNext(li)) != NULL) {
58 sds s = ln->value;
59 robj *r = createStringObject(s, sdslen(s));
60 if (rdbSaveStringObject(fp, r) == -1) return -1;
61 decrRefCount(r);
62 }
63
64 list *nrlcols = nrlind->l2;
65 if (rdbSaveLen(fp, listLength(nrlcols)) == -1) return -1;
66 li = listGetIterator(nrlcols, AL_START_HEAD);
67 while((ln = listNext(li)) != NULL) {
68 uint32 i = (uint32)(long)ln->value;
69 if (rdbSaveLen(fp, i) == -1) return -1;
70 }
71
72 return 0;
73 }
74
75 robj *rdbLoadNRL(FILE *fp) {
76 robj *iname;
77 unsigned int u;
78 d_l_t *nrlind = malloc(sizeof(d_l_t));
79 nrlind->l1 = listCreate();
80 list *nrltoks = nrlind->l1;
81 nrlind->l2 = listCreate();
82 list *nrlcols = nrlind->l2;
83
84 if ((u = rdbLoadLen(fp, NULL)) == REDIS_RDB_LENERR) return NULL;
85 nrlind->num = (int)u;
86 int imatch = nrlind->num;
87 if (!(iname = rdbLoadStringObject(fp))) return NULL;
88 if ((u = rdbLoadLen(fp, NULL)) == REDIS_RDB_LENERR) return NULL;
89 int tmatch = (int)u;
90
91 unsigned int ssize;
92 if ((ssize = rdbLoadLen(fp, NULL)) == REDIS_RDB_LENERR) return NULL;
93 for (uint32 i = 0; i < ssize; i++) {
94 robj *r;
95 if (!(r = rdbLoadStringObject(fp))) return NULL;
96 listAddNodeTail(nrltoks, sdsdup(r->ptr));
97 decrRefCount(r);
98 }
99
100 if ((ssize = rdbLoadLen(fp, NULL)) == REDIS_RDB_LENERR) return NULL;
101 for (uint32 i = 0; i < ssize; i++) {
102 uint32 col;
103 if ((col = rdbLoadLen(fp, NULL)) == REDIS_RDB_LENERR) return NULL;
104 listAddNodeTail(nrlcols, (void *)(long)col);
105 }
106 Index[server.dbid][imatch].obj = iname;
107 Index[server.dbid][imatch].table = tmatch;
108 Index[server.dbid][imatch].column = -1;
109 Index[server.dbid][imatch].type = COL_TYPE_NONE;
110 Index[server.dbid][imatch].virt = 0;
111 Index[server.dbid][imatch].nrl = 1;
112 int dbid = server.dbid;
113 if (Num_indx[dbid] < (imatch + 1)) Num_indx[dbid] = imatch + 1;
114
115 return createObject(REDIS_NRL_INDEX, nrlind);
116 }
117
183680d First code checkin
sammy authored Aug 24, 2010
118 static int rdbSaveRow(FILE *fp, bt *btr, bt_n *x) {
119 for (int i = 0; i < x->n; i++) {
120 uchar *stream = KEYS(btr, x)[i];
121 int ssize = getStreamMallocSize(stream, REDIS_ROW, btr->is_index);
122 if (rdbSaveLen(fp, ssize) == -1) return -1;
123 if (fwrite(stream, ssize, 1, fp) == 0) return -1;
124 }
125
126 if (!x->leaf) {
127 for (int i = 0; i <= x->n; i++) {
128 if (rdbSaveRow(fp, btr, NODES(btr, x)[i]) == -1) return -1;
129 }
130 }
131 return 0;
132 }
133
134 int rdbSaveBT(FILE *fp, robj *o) {
135 struct btree *btr = (struct btree *)(o->ptr);
136 if (!btr) {
137 if (fwrite(&VIRTUAL_INDEX_TYPE, 1, 1, fp) == 0) return -1;
138 return 0;
139 }
140
141 if (fwrite(&(btr->is_index), 1, 1, fp) == 0) return -1;
142 if (rdbSaveLen(fp, btr->num) == -1) return -1;
143
ef35f30 Tbl[] refactored
sammy authored Oct 1, 2010
144 int tmatch = btr->num;
75ade31 Index[] now respects dbid .... namespaces in SQL work
sammy authored Oct 2, 2010
145 int dbid = server.dbid;
183680d First code checkin
sammy authored Aug 24, 2010
146 if (btr->is_index == BTREE_TABLE) {
147 //RL4 "%d: saving table: %s virt_index: %d",
7f422d7 additional dimension to Tbl[]
sammy authored Oct 2, 2010
148 //tmatch, Tbl[dbid][tmatch].name->ptr, Tbl[dbid][tmatch].virt_indx);
149 if (rdbSaveLen(fp, Tbl[dbid][tmatch].virt_indx) == -1) return -1;
150 if (rdbSaveStringObject(fp, Tbl[dbid][tmatch].name) == -1) return -1;
151 if (rdbSaveLen(fp, Tbl[dbid][tmatch].col_count) == -1) return -1;
152 for (int i = 0; i < Tbl[dbid][tmatch].col_count; i++) {
153 if (rdbSaveStringObject(fp, Tbl[dbid][tmatch].col_name[i]) == -1)
183680d First code checkin
sammy authored Aug 24, 2010
154 return -1;
7f422d7 additional dimension to Tbl[]
sammy authored Oct 2, 2010
155 if (rdbSaveLen(fp, (int)Tbl[dbid][tmatch].col_type[i]) == -1)
183680d First code checkin
sammy authored Aug 24, 2010
156 return -1;
157 }
158 if (fwrite(&(btr->ktype), 1, 1, fp) == 0) return -1;
159 if (rdbSaveLen(fp, btr->numkeys) == -1) return -1;
160 if (btr->root && btr->numkeys > 0) {
161 if (rdbSaveRow(fp, btr, btr->root) == -1) return -1;
162 }
163 } else { //index
164 //RL4 "%d: save index: %s tbl: %d col: %d type: %d",
75ade31 Index[] now respects dbid .... namespaces in SQL work
sammy authored Oct 2, 2010
165 //tmatch, Index[dbid][tmatch].obj->ptr, Index[dbid][tmatch].table,
166 //Index[dbid][tmatch].column, Index[dbid][tmatch].type);
167 if (rdbSaveStringObject(fp, Index[dbid][tmatch].obj) == -1) return -1;
168 if (rdbSaveLen(fp, Index[dbid][tmatch].table) == -1) return -1;
169 if (rdbSaveLen(fp, Index[dbid][tmatch].column) == -1) return -1;
170 if (rdbSaveLen(fp, (int)Index[dbid][tmatch].type) == -1) return -1;
183680d First code checkin
sammy authored Aug 24, 2010
171 if (fwrite(&(btr->ktype), 1, 1, fp) == 0) return -1;
172 }
173 return 0;
174 }
175
176 static int rdbLoadRow(FILE *fp, bt *btr) {
177 unsigned int ssize;
178 if ((ssize = rdbLoadLen(fp, NULL)) == REDIS_RDB_LENERR) return -1;
179 char *bt_val = bt_malloc(ssize, btr); // mem bookkeeping done in BT
180 if (fread(bt_val, ssize, 1, fp) == 0) return -1;
181 bt_insert(btr, bt_val);
182 return 0;
183 }
184
185 //TODO minimize malloc defragmentation HERE as we know the size of each row
186 robj *rdbLoadBT(FILE *fp, redisDb *db) {
187 unsigned int u;
188 unsigned char is_index;
189 robj *o = NULL;
190 if (fread(&is_index, 1, 1, fp) == 0) return NULL;
191 if (is_index == VIRTUAL_INDEX_TYPE) {
192 return createEmptyBtreeObject();
193 }
194
195 if ((u = rdbLoadLen(fp, NULL)) == REDIS_RDB_LENERR) return NULL;
ef35f30 Tbl[] refactored
sammy authored Oct 1, 2010
196 int tmatch = (int)u;
7f422d7 additional dimension to Tbl[]
sammy authored Oct 2, 2010
197 int dbid = server.dbid;
183680d First code checkin
sammy authored Aug 24, 2010
198
199 if (is_index == BTREE_TABLE) {
200 if ((u = rdbLoadLen(fp, NULL)) == REDIS_RDB_LENERR) return NULL;
201 int inum = u;
7f422d7 additional dimension to Tbl[]
sammy authored Oct 2, 2010
202 Tbl[dbid][tmatch].virt_indx = inum;
75ade31 Index[] now respects dbid .... namespaces in SQL work
sammy authored Oct 2, 2010
203 Index[server.dbid][inum].virt = 1;
db3f032 digging the rabbit hole very deep - the non relational index .... sti…
sammy authored Oct 20, 2010
204 Index[server.dbid][inum].nrl = 0;
75ade31 Index[] now respects dbid .... namespaces in SQL work
sammy authored Oct 2, 2010
205 Index[server.dbid][inum].table = tmatch;
206 Index[server.dbid][inum].column = 0;
7f422d7 additional dimension to Tbl[]
sammy authored Oct 2, 2010
207 if (!(Tbl[dbid][tmatch].name = rdbLoadStringObject(fp))) return NULL;
183680d First code checkin
sammy authored Aug 24, 2010
208 if ((u = rdbLoadLen(fp, NULL)) == REDIS_RDB_LENERR) return NULL;
7f422d7 additional dimension to Tbl[]
sammy authored Oct 2, 2010
209 Tbl[dbid][tmatch].col_count = u;
210 for (int i = 0; i < Tbl[dbid][tmatch].col_count; i++) {
211 if (!(Tbl[dbid][tmatch].col_name[i] = rdbLoadStringObject(fp)))
183680d First code checkin
sammy authored Aug 24, 2010
212 return NULL;
213 if ((u = rdbLoadLen(fp, NULL)) == REDIS_RDB_LENERR) return NULL;
7f422d7 additional dimension to Tbl[]
sammy authored Oct 2, 2010
214 Tbl[dbid][tmatch].col_type[i] = (unsigned char)u;
183680d First code checkin
sammy authored Aug 24, 2010
215 }
216
75ade31 Index[] now respects dbid .... namespaces in SQL work
sammy authored Oct 2, 2010
217 Index[server.dbid][inum].type = Tbl[dbid][tmatch].col_type[0];
218 Index[server.dbid][inum].obj =
219 createStringObject(Tbl[dbid][tmatch].name->ptr,
220 sdslen(Tbl[dbid][tmatch].name->ptr));
221 Index[server.dbid][inum].obj->ptr =
222 sdscatprintf(Index[server.dbid][inum].obj->ptr, "%s%s%s%s", COLON,
223 (char *)Tbl[dbid][tmatch].col_name[0]->ptr,
224 COLON, INDEX_DELIM);
225 dictAdd(db->dict, Index[server.dbid][inum].obj, NULL);
7f422d7 additional dimension to Tbl[]
sammy authored Oct 2, 2010
226 if (Num_indx[dbid] < (inum + 1)) {
227 Num_indx[dbid] = inum + 1;
036c922 ruby client showed some bugs
sammy authored Sep 30, 2010
228 }
183680d First code checkin
sammy authored Aug 24, 2010
229
230 unsigned char ktype;
231 if (fread(&ktype, 1, 1, fp) == 0) return NULL;
232
ef35f30 Tbl[] refactored
sammy authored Oct 1, 2010
233 o = createBtreeObject(ktype, tmatch, is_index);
183680d First code checkin
sammy authored Aug 24, 2010
234 struct btree *btr = (struct btree *)(o->ptr);
235
236 unsigned int bt_num;
237 if ((bt_num = rdbLoadLen(fp, NULL)) == REDIS_RDB_LENERR) return NULL;
238
239 for (int unsigned i = 0; i < bt_num; i++) {
240 if (rdbLoadRow(fp, btr) == -1) return NULL;
241 }
242
db3f032 digging the rabbit hole very deep - the non relational index .... sti…
sammy authored Oct 20, 2010
243 if (Num_tbls[dbid] < (tmatch + 1)) Num_tbls[dbid] = tmatch + 1;
183680d First code checkin
sammy authored Aug 24, 2010
244 } else { /* BTREE_INDEX */
db3f032 digging the rabbit hole very deep - the non relational index .... sti…
sammy authored Oct 20, 2010
245 int imatch = tmatch;
246 Index[server.dbid][imatch].nrl = 0;
247 Index[server.dbid][imatch].obj = rdbLoadStringObject(fp);
248 if (!(Index[server.dbid][imatch].obj)) return NULL;
183680d First code checkin
sammy authored Aug 24, 2010
249 if ((u = rdbLoadLen(fp, NULL)) == REDIS_RDB_LENERR) return NULL;
db3f032 digging the rabbit hole very deep - the non relational index .... sti…
sammy authored Oct 20, 2010
250 Index[server.dbid][imatch].table = (int)u;
183680d First code checkin
sammy authored Aug 24, 2010
251 if ((u = rdbLoadLen(fp, NULL)) == REDIS_RDB_LENERR) return NULL;
db3f032 digging the rabbit hole very deep - the non relational index .... sti…
sammy authored Oct 20, 2010
252 Index[server.dbid][imatch].column = (int)u;
183680d First code checkin
sammy authored Aug 24, 2010
253 if ((u = rdbLoadLen(fp, NULL)) == REDIS_RDB_LENERR) return NULL;
db3f032 digging the rabbit hole very deep - the non relational index .... sti…
sammy authored Oct 20, 2010
254 Index[server.dbid][imatch].type = (unsigned char)u;
183680d First code checkin
sammy authored Aug 24, 2010
255 unsigned char ktype;
256 if (fread(&ktype, 1, 1, fp) == 0) return NULL;
db3f032 digging the rabbit hole very deep - the non relational index .... sti…
sammy authored Oct 20, 2010
257 o = createBtreeObject(ktype, imatch, is_index);
258 Index[server.dbid][imatch].virt = 0;
259 if (Num_indx[dbid] < (imatch + 1)) Num_indx[dbid] = imatch + 1;
183680d First code checkin
sammy authored Aug 24, 2010
260 }
261 return o;
262 }
263
db3f032 digging the rabbit hole very deep - the non relational index .... sti…
sammy authored Oct 20, 2010
264 static void runNrlIndexFromStream(uchar *stream,
265 d_l_t *nrlind,
266 int itbl) {
267 robj key, val;
268 assignKeyRobj(stream, &key);
269 assignValRobj(stream, REDIS_ROW, &val, BTREE_TABLE);
270 /* create command and run it */
271 sds cmd = genNRL_Cmd(nrlind, &key, NULL, NULL, 0, &val, itbl);
272 runCmdInFakeClient(cmd);
273 sdsfree(cmd);
274 if (key.encoding == REDIS_ENCODING_RAW) {
275 sdsfree(key.ptr); /* free from assignKeyRobj sflag[1,4] */
276 }
277 }
278 static void makeIndexFromStream(uchar *stream,
279 bt *ibtr,
280 int icol,
281 int itbl) {
183680d First code checkin
sammy authored Aug 24, 2010
282 robj key, val;
283 assignKeyRobj(stream, &key);
284 assignValRobj(stream, REDIS_ROW, &val, ibtr->is_index);
db3f032 digging the rabbit hole very deep - the non relational index .... sti…
sammy authored Oct 20, 2010
285 /* get the pk and the fk and then call iAdd() */
286 robj *fk = createColObjFromRow(&val, icol, &key, itbl); /* freeME */
7f422d7 additional dimension to Tbl[]
sammy authored Oct 2, 2010
287 iAdd(ibtr, fk, &key, Tbl[server.dbid][itbl].col_type[0]);
183680d First code checkin
sammy authored Aug 24, 2010
288 decrRefCount(fk);
289 if (key.encoding == REDIS_ENCODING_RAW) {
290 sdsfree(key.ptr); /* free from assignKeyRobj sflag[1,4] */
291 }
292 }
293
db3f032 digging the rabbit hole very deep - the non relational index .... sti…
sammy authored Oct 20, 2010
294 int buildIndex(bt *btr, bt_n *x, bt *ibtr, int icol, int itbl, bool nrl) {
183680d First code checkin
sammy authored Aug 24, 2010
295 for (int i = 0; i < x->n; i++) {
296 uchar *stream = KEYS(btr, x)[i];
db3f032 digging the rabbit hole very deep - the non relational index .... sti…
sammy authored Oct 20, 2010
297 if (nrl) runNrlIndexFromStream(stream, (d_l_t *)ibtr, itbl);
298 else makeIndexFromStream(stream, ibtr, icol, itbl);
183680d First code checkin
sammy authored Aug 24, 2010
299 }
300
301 if (!x->leaf) {
302 for (int i = 0; i <= x->n; i++) {
db3f032 digging the rabbit hole very deep - the non relational index .... sti…
sammy authored Oct 20, 2010
303 buildIndex(btr, NODES(btr, x)[i], ibtr, icol, itbl, nrl);
183680d First code checkin
sammy authored Aug 24, 2010
304 }
305 }
306 return 0;
307 }
308
309 void rdbLoadFinished(redisDb *db) {
7f422d7 additional dimension to Tbl[]
sammy authored Oct 2, 2010
310 for (int i = 0; i < Num_indx[server.dbid]; i++) {
75ade31 Index[] now respects dbid .... namespaces in SQL work
sammy authored Oct 2, 2010
311 if (Index[server.dbid][i].virt) continue;
db3f032 digging the rabbit hole very deep - the non relational index .... sti…
sammy authored Oct 20, 2010
312 if (Index[server.dbid][i].nrl) continue; /* on rebild nrlind is NOOP */
75ade31 Index[] now respects dbid .... namespaces in SQL work
sammy authored Oct 2, 2010
313 robj *ind = Index[server.dbid][i].obj;
183680d First code checkin
sammy authored Aug 24, 2010
314 if (!ind) continue;
315 robj *ibt = lookupKey(db, ind);
316 bt *ibtr = (struct btree *)(ibt->ptr);
75ade31 Index[] now respects dbid .... namespaces in SQL work
sammy authored Oct 2, 2010
317 int itbl = Index[server.dbid][i].table;
318 int icol = Index[server.dbid][i].column;
7f422d7 additional dimension to Tbl[]
sammy authored Oct 2, 2010
319 robj *o = lookupKey(db, Tbl[server.dbid][itbl].name);
183680d First code checkin
sammy authored Aug 24, 2010
320 bt *btr = (struct btree *)(o->ptr);
db3f032 digging the rabbit hole very deep - the non relational index .... sti…
sammy authored Oct 20, 2010
321 buildIndex(btr, btr->root, ibtr, icol, itbl, 0);
183680d First code checkin
sammy authored Aug 24, 2010
322 #if 0
323 struct btree *ibtr = (struct btree *)(ibt->ptr);
324 RL4 "INDEX: %d", ibtr->num);
325 bt_dumptree(ibtr, 0, 0);
326 #endif
327 }
328 }
329
Something went wrong with that request. Please try again.