Skip to content
This repository
Browse code

Merge branch 'writing'

  • Loading branch information...
commit 174ea558640a04c9ee83a354a7bd454d6e02b59b 2 parents d891a4f + 5156e68
Michael Stephens authored September 28, 2009
2  SConstruct
@@ -39,6 +39,6 @@ conf.CheckLib( makeBoost( "system" ) )
39 39
 
40 40
 env = conf.Finish()
41 41
 
42  
-files = ['main.cpp', 'operations.cpp', 'options.cpp']
  42
+files = ['main.cpp', 'operations.cpp', 'options.cpp', 'local_gridfile.cpp']
43 43
 
44 44
 env.Program('mount_gridfs', files)
67  local_gridfile.cpp
... ...
@@ -0,0 +1,67 @@
  1
+#include "local_gridfile.h"
  2
+
  3
+#include <algorithm>
  4
+
  5
+using namespace std;
  6
+
  7
+int LocalGridFile::write(const char *buf, size_t nbyte, off_t offset)
  8
+{
  9
+    int last_chunk = (offset + nbyte) / _chunkSize;
  10
+    int written = 0;
  11
+
  12
+    while(last_chunk > _chunks.size() - 1) {
  13
+        char *new_buf = new char[_chunkSize];
  14
+        memset(new_buf, 0, _chunkSize);
  15
+        _chunks.push_back(new_buf);
  16
+    }
  17
+
  18
+    int chunk_num = offset / _chunkSize;
  19
+    char* dest_buf = _chunks[chunk_num];
  20
+
  21
+    int buf_offset = offset % _chunkSize;
  22
+    if(buf_offset) {
  23
+        dest_buf += offset % _chunkSize;
  24
+        int to_write = min(nbyte - written,
  25
+                           (long unsigned int)(_chunkSize - buf_offset));
  26
+        memcpy(dest_buf, buf, to_write);
  27
+        written += to_write;
  28
+        chunk_num++;
  29
+    }
  30
+
  31
+    while(written < nbyte) {
  32
+        dest_buf = _chunks[chunk_num];
  33
+        int to_write = min(nbyte - written,
  34
+                           (long unsigned int)_chunkSize);
  35
+        memcpy(dest_buf, buf, to_write);
  36
+        written += to_write;
  37
+        chunk_num++;
  38
+    }
  39
+
  40
+    _length = max(_length, (int)offset + written);
  41
+    _dirty = true;
  42
+    
  43
+    return written;
  44
+}
  45
+
  46
+int LocalGridFile::read(char* buf, size_t size, off_t offset)
  47
+{
  48
+    size_t len = 0;
  49
+    int chunk_num = offset / _chunkSize;
  50
+
  51
+    while(len < size && chunk_num < _chunks.size()) {
  52
+        const char* chunk = _chunks[chunk_num];
  53
+        size_t to_read = min((size_t)_chunkSize, size - len);
  54
+
  55
+        if(!len && offset) {
  56
+            chunk += offset % _chunkSize;
  57
+            to_read = min(to_read,
  58
+                          (size_t)(_chunkSize - (offset % _chunkSize)));
  59
+        }
  60
+
  61
+        memcpy(buf + len, chunk, to_read);
  62
+        len += to_read;
  63
+        chunk_num++;
  64
+    }
  65
+
  66
+    return len;
  67
+}
40  local_gridfile.h
... ...
@@ -0,0 +1,40 @@
  1
+#ifndef _LOCAL_GRIDFILE_H
  2
+#define _LOCAL_GRIDFILE_H
  3
+
  4
+#include <vector>
  5
+#include <cstring>
  6
+#include <iostream>
  7
+
  8
+const unsigned int DEFAULT_CHUNK_SIZE = 256 * 1024;
  9
+
  10
+class LocalGridFile {
  11
+public:
  12
+    LocalGridFile(int chunkSize = DEFAULT_CHUNK_SIZE) :
  13
+    _chunkSize(chunkSize), _length(0), _dirty(true) {
  14
+          _chunks.push_back(new char[_chunkSize]);
  15
+      }
  16
+
  17
+    ~LocalGridFile() {
  18
+        for(std::vector<char*>::iterator i = _chunks.begin();
  19
+            i != _chunks.end(); i++) {
  20
+            delete *i;
  21
+        }
  22
+    }
  23
+
  24
+    int getChunkSize() { return _chunkSize; }
  25
+    int getNumChunks() { return _chunks.size(); }
  26
+    int getLength() { return _length; }
  27
+    char* getChunk(int n) { return _chunks[n]; }
  28
+    bool dirty() { return _dirty; }
  29
+    void flushed() { _dirty = false; }
  30
+
  31
+    int write(const char* buf, size_t nbyte, off_t offset);
  32
+    int read(char* buf, size_t size, off_t offset);
  33
+
  34
+private:
  35
+    int _chunkSize, _length;
  36
+    bool _dirty;
  37
+    std::vector<char*> _chunks;
  38
+};
  39
+
  40
+#endif
6  main.cpp
@@ -28,10 +28,16 @@ int main(int argc, char *argv[])
28 28
     gridfs_oper.getattr = gridfs_getattr;
29 29
     gridfs_oper.readdir = gridfs_readdir;
30 30
     gridfs_oper.open = gridfs_open;
  31
+    gridfs_oper.create = gridfs_create;
  32
+    gridfs_oper.release = gridfs_release;
  33
+    gridfs_oper.unlink = gridfs_unlink;
31 34
     gridfs_oper.read = gridfs_read;
32 35
     gridfs_oper.listxattr = gridfs_listxattr;
33 36
     gridfs_oper.getxattr = gridfs_getxattr;
34 37
     gridfs_oper.setxattr = gridfs_setxattr;
  38
+    gridfs_oper.write = gridfs_write;
  39
+    gridfs_oper.flush = gridfs_flush;
  40
+    gridfs_oper.rename = gridfs_rename;
35 41
 
36 42
     struct fuse_args args = FUSE_ARGS_INIT(argc, argv);
37 43
 
251  operations.cpp
@@ -18,6 +18,7 @@
18 18
 #include "operations.h"
19 19
 #include "options.h"
20 20
 #include "utils.h"
  21
+#include "local_gridfile.h"
21 22
 #include <algorithm>
22 23
 #include <cstring>
23 24
 #include <cerrno>
@@ -34,35 +35,52 @@
34 35
 using namespace std;
35 36
 using namespace mongo;
36 37
 
  38
+std::map<string, LocalGridFile*> open_files;
  39
+
  40
+unsigned int FH;
  41
+
37 42
 int gridfs_getattr(const char *path, struct stat *stbuf)
38 43
 {
39 44
     int res = 0;
40 45
     memset(stbuf, 0, sizeof(struct stat));
41 46
     
42 47
     if(strcmp(path, "/") == 0) {
43  
-        stbuf->st_mode = S_IFDIR | 0555;
  48
+        stbuf->st_mode = S_IFDIR | 0777;
44 49
         stbuf->st_nlink = 2;
45  
-    } else {
46  
-        path = fuse_to_mongo_path(path);
  50
+        return 0;
  51
+    }
47 52
 
48  
-        ScopedDbConnection sdc(gridfs_options.host);
49  
-        GridFS gf(sdc.conn(), gridfs_options.db);
50  
-        GridFile file = gf.findFile(path);
51  
-        sdc.done();
  53
+    path = fuse_to_mongo_path(path);
52 54
 
53  
-        if(!file.exists()) {
54  
-            return -ENOENT;
55  
-        }
  55
+    map<string,LocalGridFile*>::const_iterator file_iter;
  56
+    file_iter = open_files.find(path);
56 57
 
57  
-        stbuf->st_mode = S_IFREG | 0555;
  58
+    if(file_iter != open_files.end()) {
  59
+        stbuf->st_mode = S_IFREG | 0777;
58 60
         stbuf->st_nlink = 1;
59  
-        stbuf->st_size = file.getContentLength();
  61
+        stbuf->st_ctime = time(NULL);
  62
+        stbuf->st_mtime = time(NULL);
  63
+        stbuf->st_size = file_iter->second->getLength();
  64
+        return 0;
  65
+    }
60 66
 
61  
-        time_t upload_time = mongo_time_to_unix_time(file.getUploadDate());
62  
-        stbuf->st_ctime = upload_time;
63  
-        stbuf->st_mtime = upload_time;
  67
+    ScopedDbConnection sdc(gridfs_options.host);
  68
+    GridFS gf(sdc.conn(), gridfs_options.db);
  69
+    GridFile file = gf.findFile(path);
  70
+    sdc.done();
  71
+
  72
+    if(!file.exists()) {
  73
+        return -ENOENT;
64 74
     }
65 75
 
  76
+    stbuf->st_mode = S_IFREG | 0555;
  77
+    stbuf->st_nlink = 1;
  78
+    stbuf->st_size = file.getContentLength();
  79
+
  80
+    time_t upload_time = mongo_time_to_unix_time(file.getUploadDate());
  81
+    stbuf->st_ctime = upload_time;
  82
+    stbuf->st_mtime = upload_time;
  83
+
66 84
     return 0;
67 85
 }
68 86
 
@@ -85,27 +103,76 @@ int gridfs_readdir(const char *path, void *buf, fuse_fill_dir_t filler,
85 103
     }
86 104
     sdc.done();
87 105
 
  106
+    for(map<string,LocalGridFile*>::const_iterator i = open_files.begin();
  107
+        i != open_files.end(); i++)
  108
+    {
  109
+        filler(buf, i->first.c_str(), NULL, 0);
  110
+    }
  111
+
88 112
     return 0;
89 113
 }
90 114
 
91 115
 int gridfs_open(const char *path, struct fuse_file_info *fi)
92 116
 {
93  
-    if((fi->flags & O_ACCMODE) != O_RDONLY) {
  117
+    path = fuse_to_mongo_path(path);
  118
+
  119
+    if((fi->flags & O_ACCMODE) == O_RDONLY) {
  120
+        if(open_files.find(path) != open_files.end()) {
  121
+            return 0;
  122
+        }
  123
+
  124
+        ScopedDbConnection sdc(gridfs_options.host);
  125
+        GridFS gf(sdc.conn(), gridfs_options.db);
  126
+        GridFile file = gf.findFile(path);
  127
+        sdc.done();
  128
+
  129
+        if(file.exists()) {
  130
+            return 0;
  131
+        }
  132
+
  133
+        return -ENOENT;
  134
+    } else {
94 135
         return -EACCES;
95 136
     }
  137
+}
96 138
 
  139
+int gridfs_create(const char* path, mode_t mode, struct fuse_file_info* ffi)
  140
+{
97 141
     path = fuse_to_mongo_path(path);
98 142
 
99  
-    ScopedDbConnection sdc(gridfs_options.host);
100  
-    GridFS gf(sdc.conn(), gridfs_options.db);
101  
-    GridFile file = gf.findFile(path);
102  
-    sdc.done();
  143
+    open_files[path] = new LocalGridFile(DEFAULT_CHUNK_SIZE);
  144
+
  145
+    ffi->fh = FH++;
  146
+
  147
+    return 0;
  148
+}
  149
+
  150
+int gridfs_release(const char* path, struct fuse_file_info* ffi)
  151
+{
  152
+    path = fuse_to_mongo_path(path);
103 153
 
104  
-    if(file.exists()) {
  154
+    if(!ffi->fh) {
  155
+        // fh is not set if file is opened read only
  156
+        // Would check ffi->flags for O_RDONLY instead but MacFuse doesn't
  157
+        // seem to properly pass flags into release
105 158
         return 0;
106 159
     }
107 160
 
108  
-    return -ENOENT;
  161
+    delete open_files[path];
  162
+    open_files.erase(path);
  163
+
  164
+    return 0;
  165
+}
  166
+
  167
+int gridfs_unlink(const char* path) {
  168
+    path = fuse_to_mongo_path(path);
  169
+
  170
+    ScopedDbConnection sdc(gridfs_options.host);
  171
+    GridFS gf(sdc.conn(), gridfs_options.db);
  172
+    gf.removeFile(path);
  173
+    sdc.done();
  174
+
  175
+    return 0;
109 176
 }
110 177
 
111 178
 int gridfs_read(const char *path, char *buf, size_t size, off_t offset,
@@ -114,6 +181,13 @@ int gridfs_read(const char *path, char *buf, size_t size, off_t offset,
114 181
     path = fuse_to_mongo_path(path);
115 182
     size_t len = 0;
116 183
 
  184
+    map<string,LocalGridFile*>::const_iterator file_iter;
  185
+    file_iter = open_files.find(path);
  186
+    if(file_iter != open_files.end()) {
  187
+        LocalGridFile *lgf = file_iter->second;
  188
+        return lgf->read(buf, size, offset);
  189
+    }
  190
+
117 191
     ScopedDbConnection sdc(gridfs_options.host);
118 192
     GridFS gf(sdc.conn(), gridfs_options.db);
119 193
     GridFile file = gf.findFile(path);
@@ -153,6 +227,10 @@ int gridfs_listxattr(const char* path, char* list, size_t size)
153 227
 {
154 228
     path = fuse_to_mongo_path(path);
155 229
 
  230
+    if(open_files.find(path) != open_files.end()) {
  231
+        return 0;
  232
+    }
  233
+
156 234
     ScopedDbConnection sdc(gridfs_options.host);
157 235
     GridFS gf(sdc.conn(), gridfs_options.db);
158 236
     GridFile file = gf.findFile(path);
@@ -197,6 +275,10 @@ int gridfs_getxattr(const char* path, const char* name, char* value, size_t size
197 275
         return -ENOATTR;
198 276
     }
199 277
 
  278
+    if(open_files.find(path) != open_files.end()) {
  279
+        return -ENOATTR;
  280
+    }
  281
+
200 282
     ScopedDbConnection sdc(gridfs_options.host);
201 283
     GridFS gf(sdc.conn(), gridfs_options.db);
202 284
     GridFile file = gf.findFile(path);
@@ -234,3 +316,128 @@ int gridfs_setxattr(const char* path, const char* name, const char* value,
234 316
 {
235 317
     return -ENOTSUP;
236 318
 }
  319
+
  320
+int gridfs_write(const char* path, const char* buf, size_t nbyte,
  321
+                 off_t offset, struct fuse_file_info* ffi)
  322
+{
  323
+    path = fuse_to_mongo_path(path);
  324
+
  325
+    if(open_files.find(path) == open_files.end()) {
  326
+        return -ENOENT;
  327
+    }
  328
+
  329
+    LocalGridFile *lgf = open_files[path];
  330
+
  331
+    return lgf->write(buf, nbyte, offset);
  332
+}
  333
+
  334
+int gridfs_flush(const char* path, struct fuse_file_info *ffi)
  335
+{
  336
+    path = fuse_to_mongo_path(path);
  337
+
  338
+    string db_name = gridfs_options.db;
  339
+
  340
+    map<string,LocalGridFile*>::iterator file_iter;
  341
+    file_iter = open_files.find(path);
  342
+    if(file_iter == open_files.end()) {
  343
+        return -ENOENT;
  344
+    }
  345
+
  346
+    LocalGridFile *lgf = file_iter->second;
  347
+
  348
+    if(!lgf->dirty()) {
  349
+        return 0;
  350
+    }
  351
+
  352
+    ScopedDbConnection sdc(gridfs_options.host);
  353
+    DBClientBase &client = sdc.conn();
  354
+    GridFS gf(sdc.conn(), gridfs_options.db);
  355
+
  356
+    if(gf.findFile(path).exists()) {
  357
+        gf.removeFile(path);
  358
+    }
  359
+
  360
+    BSONObjBuilder fileObject;
  361
+    BSONObj idObj;
  362
+    OID id;
  363
+    {
  364
+        id.init();
  365
+        fileObject.appendOID("_id", &id);
  366
+        fileObject << "filename" << path;
  367
+        fileObject << "chunkSize" << lgf->getChunkSize();
  368
+        fileObject << "length" << lgf->getLength();
  369
+        fileObject.appendDate("uploadDate", mongo_time());
  370
+
  371
+        BSONObjBuilder b;
  372
+        b.appendOID("_id", &id);
  373
+        idObj = b.obj();
  374
+    }
  375
+
  376
+    for(int i = 0; i < lgf->getNumChunks(); i++) {
  377
+        const char *buf = lgf->getChunk(i);
  378
+
  379
+         int len = min(lgf->getChunkSize(), lgf->getLength() - i * lgf->getChunkSize());
  380
+
  381
+        OID chunk_id;
  382
+        chunk_id.init();
  383
+
  384
+        BSONObjBuilder chunk_b;
  385
+        chunk_b.appendOID("_id", &chunk_id);
  386
+        chunk_b.appendOID("files_id", &id);
  387
+        chunk_b.append("n", i);
  388
+        chunk_b.appendBinDataArray("data", buf, len);
  389
+        client.insert(db_name + ".fs.chunks", chunk_b.obj());
  390
+    }
  391
+
  392
+    BSONObj res;
  393
+    client.runCommand(gridfs_options.db, BSON("filemd5" << id), res);
  394
+    fileObject.appendAs(res["md5"], "md5");
  395
+
  396
+    BSONObj real = fileObject.obj();
  397
+    client.insert(db_name + ".fs.files", real);
  398
+
  399
+    sdc.done();
  400
+
  401
+    lgf->flushed();
  402
+
  403
+    return 0;
  404
+}
  405
+
  406
+int gridfs_rename(const char* old_path, const char* new_path)
  407
+{
  408
+    old_path = fuse_to_mongo_path(old_path);
  409
+    new_path = fuse_to_mongo_path(new_path);
  410
+
  411
+    ScopedDbConnection sdc(gridfs_options.host);
  412
+    DBClientBase &client = sdc.conn();
  413
+
  414
+    string db_name = gridfs_options.db;
  415
+
  416
+    BSONObj file_obj = client.findOne(db_name + ".fs.files",
  417
+                                      BSON("filename" << old_path));
  418
+
  419
+    if(file_obj.isEmpty()) {
  420
+        return -ENOENT;
  421
+    }
  422
+
  423
+    BSONObjBuilder b;
  424
+    set<string> field_names;
  425
+    file_obj.getFieldNames(field_names);
  426
+
  427
+    for(set<string>::iterator name = field_names.begin();
  428
+        name != field_names.end(); name++)
  429
+    {
  430
+        if(*name != "filename") {
  431
+            b.append(file_obj.getField(*name));
  432
+        }
  433
+    }
  434
+
  435
+    b << "filename" << new_path;
  436
+
  437
+    client.update(db_name + ".fs.files",
  438
+                  BSON("_id" << file_obj.getField("_id")), b.obj());
  439
+
  440
+    sdc.done();
  441
+
  442
+    return 0;
  443
+}
13  operations.h
@@ -29,9 +29,15 @@ int gridfs_readdir(const char *path, void *buf, fuse_fill_dir_t filler,
29 29
 
30 30
 int gridfs_open(const char *path, struct fuse_file_info *fi);
31 31
 
  32
+int gridfs_create(const char* path, mode_t mode, struct fuse_file_info* ffi);
  33
+
  34
+int gridfs_release(const char* path, struct fuse_file_info* ffi);
  35
+
32 36
 int gridfs_read(const char *path, char *buf, size_t size, off_t offset,
33 37
                 struct fuse_file_info *fi);
34 38
 
  39
+int gridfs_unlink(const char* path);
  40
+
35 41
 int gridfs_listxattr(const char* path, char* list, size_t size);
36 42
 
37 43
 int gridfs_getxattr(const char* path, const char* name, char* value, size_t size);
@@ -39,4 +45,11 @@ int gridfs_getxattr(const char* path, const char* name, char* value, size_t size
39 45
 int gridfs_setxattr(const char* path, const char* name, const char* value,
40 46
                     size_t size, int flags);
41 47
 
  48
+int gridfs_write(const char* path, const char* buf, size_t nbyte,
  49
+                 off_t offset, struct fuse_file_info* ffi);
  50
+
  51
+int gridfs_flush(const char* path, struct fuse_file_info* ffi);
  52
+
  53
+int gridfs_rename(const char* old_path, const char* new_path);
  54
+
42 55
 #endif
10  utils.h
@@ -36,6 +36,16 @@ inline time_t mongo_time_to_unix_time(unsigned long long mtime)
36 36
     return mtime / 1000;
37 37
 }
38 38
 
  39
+inline time_t unix_time_to_mongo_time(unsigned long long utime)
  40
+{
  41
+    return utime * 1000;
  42
+}
  43
+
  44
+inline time_t mongo_time()
  45
+{
  46
+    return unix_time_to_mongo_time(time(NULL));
  47
+}
  48
+
39 49
 inline std::string namespace_xattr(const std::string name)
40 50
 {
41 51
 #ifdef __linux__

0 notes on commit 174ea55

Please sign in to comment.
Something went wrong with that request. Please try again.