Permalink
Browse files

simple-mmap - Simple readonly mmap()'ed files in ruby

  • Loading branch information...
0 parents commit 3baee14a0515e2c4385133f93bb87a70413f19d8 @js js committed Mar 29, 2009
Showing with 263 additions and 0 deletions.
  1. +10 −0 .gitignore
  2. +6 −0 History.txt
  3. +7 −0 Manifest.txt
  4. +51 −0 README.txt
  5. +35 −0 Rakefile
  6. +5 −0 ext/extconf.rb
  7. +116 −0 ext/mapped_file.c
  8. +3 −0 lib/simple_mmap.rb
  9. +3 −0 lib/simple_mmap/version.rb
  10. +27 −0 test/test_mapped_file.rb
@@ -0,0 +1,10 @@
+/ext/*.o
+/ext/*.bundle
+/ext/*.so
+/ext/*.dll
+/ext/mkmf.log
+/lib/simple_mmap/*.bundle
+/lib/simple_mmap/*.so
+/lib/simple_mmap/*.dll
+/ext/Makefile
+/pkg
@@ -0,0 +1,6 @@
+=== 1.0.0 / 2009-03-28
+
+* 1 major enhancement
+
+ * Birthday!
+
@@ -0,0 +1,7 @@
+History.txt
+Manifest.txt
+README.txt
+Rakefile
+bin/simple_mmap
+lib/simple_mmap.rb
+test/test_simple_mmap.rb
@@ -0,0 +1,51 @@
+= simple_mmap
+
+* FIX (url)
+
+== DESCRIPTION:
+
+ A simplistic interface for reading memory mapped files
+
+== FEATURES/PROBLEMS:
+
+* FIX (list of features or problems)
+
+== SYNOPSIS:
+
+>> fw = SimpleMmap::FileWindow.open("path/to/large/file")
+>> fw[23467, 200] #=> data between byte 23467 and the next 200 bytes
+# works with the same semantics as Array#[] (eg. fw[10..14] and fw[10...14])
+>> fw.close # you MUST close the FileWindow after you're done (HAHA no blocks for you yet!)
+
+== REQUIREMENTS:
+
+* A mmap() capable platform.
+
+== INSTALL:
+
+* FIX (sudo gem install, anything else)
+
+== LICENSE:
+
+(The MIT License)
+
+Copyright (c) 2009 Johan Sørensen
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@@ -0,0 +1,35 @@
+# encoding: utf-8
+
+require 'rubygems'
+require 'hoe'
+require './lib/simple_mmap/version.rb'
+require "rake/clean"
+
+Hoe.new('simple_mmap', SimpleMmap::VERSION) do |p|
+ p.rubyforge_name = 'simple-mmap'
+ p.developer('Johan Sørensen', 'johan@johansorensen.com')
+end
+
+DLEXT = Config::CONFIG['DLEXT']
+
+file 'ext/Makefile' => FileList['ext/{*.c,*.h,*.rb}'] do
+ chdir('ext') { ruby 'extconf.rb' }
+end
+CLEAN.include 'ext/Makefile', 'ext/mkmf.log'
+
+file "ext/mapped_file.#{DLEXT}" => FileList['ext/Makefile', 'ext/*.{c,h,rb}'] do |f|
+ sh 'cd ext && make'
+end
+CLEAN.include 'ext/*.{o,bundle,so,dll}'
+
+file "lib/simple_mmap/mapped_file.#{DLEXT}" => "ext/mapped_file.#{DLEXT}" do |f|
+ cp f.prerequisites, "lib/simple_mmap/", :preserve => true
+end
+#CLEAN.include "lib/simple_mmap/mapped_file.#{DLEXT}"
+
+desc 'Build the mapped_file extension'
+task :build => "lib/simple_mmap/mapped_file.#{DLEXT}"
+
+task :test => [:build]
+
+task :default => :test
@@ -0,0 +1,5 @@
+require 'mkmf'
+
+dir_config('mapped_file')
+have_func('mmap')
+create_makefile('mapped_file')
@@ -0,0 +1,116 @@
+#include <stdio.h>
+#include <sys/stat.h>
+#include <sys/fcntl.h>
+#include <sys/mman.h>
+#include <errno.h>
+#include <unistd.h>
+#include "ruby.h"
+
+// a memory mapped file
+typedef struct {
+ int fd;
+ caddr_t map;
+ size_t len;
+} simple_mmap_map;
+
+static VALUE mod_simple_mmap;
+static VALUE sm_mapped_file;
+static VALUE sm_map_data;
+
+static VALUE sm_mapped_file_initialize(VALUE vself, VALUE filename)
+{
+ int fd = -1;
+ size_t length;
+ struct stat st;
+ caddr_t base = (caddr_t) - 1;
+ VALUE vsm_map;
+ simple_mmap_map *sm_map;
+
+ fd = open(RSTRING_PTR(filename), O_RDONLY);
+ if (fd == -1) {
+ rb_raise(rb_eArgError, "Failed to open file %s", RSTRING_PTR(filename));
+ close(fd);
+ }
+
+ // Get the size of the file
+ if (fstat(fd, &st) == -1) {
+ rb_raise(rb_eArgError, "Failed to stat file %s", RSTRING_PTR(filename));
+ close(fd);
+ }
+ length = st.st_size;
+
+ // do the mmap
+ base = mmap(NULL, length, PROT_READ, MAP_SHARED, fd, 0);
+ if (base == (caddr_t) -1) {
+ rb_raise(rb_eArgError, "Failed to mmap file %s", RSTRING_PTR(filename));
+ close(fd);
+ }
+
+ // set the mmap structure as an ivar
+ sm_map = ALLOC(simple_mmap_map);
+ vsm_map = Data_Wrap_Struct(sm_map_data, NULL, free, sm_map);
+ sm_map->fd = fd;
+ sm_map->map = base;
+ sm_map->len = length;
+ rb_ivar_set(vself, rb_intern("@mmap_data"), vsm_map);
+
+ return Qnil;
+}
+
+static VALUE sm_mapped_file_close(VALUE vself)
+{
+ VALUE vsm_map;
+ simple_mmap_map *sm_map;
+
+ sm_map = ALLOC(simple_mmap_map);
+ vsm_map = rb_ivar_get(vself, rb_intern("@mmap_data"));
+ Data_Get_Struct(vsm_map, simple_mmap_map, sm_map);
+ // TODO: check that the mmap+fd is active first
+ munmap(sm_map->map, sm_map->len);
+ close(sm_map->fd);
+ free(sm_map);
+}
+
+static VALUE sm_mapped_file_read_window_data(VALUE vself, VALUE voffset, VALUE vlength)
+{
+ size_t offset = NUM2INT(voffset);
+ size_t length = NUM2INT(vlength);
+ char buff[length];
+ VALUE vsm_map;
+ simple_mmap_map *sm_map;
+
+ sm_map = ALLOC(simple_mmap_map);
+ vsm_map = rb_ivar_get(vself, rb_intern("@mmap_data"));
+ Data_Get_Struct(vsm_map, simple_mmap_map, sm_map);
+
+ if (offset > sm_map->len) {
+ return Qnil;
+ }
+
+ size_t curr;
+ curr = offset;
+ size_t i;
+ for(i = 0; i < length; ++i) {
+ //printf("i=%i offset=%i length=%i curr=%i map->len=%i\n", i, offset, length, curr, sm_map->len);
+ if ((curr + i) > sm_map->len)
+ break;
+ buff[i] = sm_map->map[curr++];
+ }
+
+ if ((offset + length) > sm_map->len)
+ return rb_str_new(buff, ((offset+length) - sm_map->len)-1);
+
+ return rb_str_new(buff, length);
+}
+
+void Init_mapped_file()
+{
+ mod_simple_mmap = rb_define_module("SimpleMmap");
+
+ sm_mapped_file = rb_define_class_under(mod_simple_mmap, "MappedFile", rb_cObject);
+ rb_define_private_method(sm_mapped_file, "initialize", sm_mapped_file_initialize, 1);
+ rb_define_method(sm_mapped_file, "close", sm_mapped_file_close, 0);
+ rb_define_method(sm_mapped_file, "read_window_data", sm_mapped_file_read_window_data, 2);
+
+ sm_map_data = rb_define_class_under(sm_mapped_file, "MmapData", rb_cObject);
+}
@@ -0,0 +1,3 @@
+$:.unshift File.dirname(__FILE__)
+require "simple_mmap/version"
+require "simple_mmap/mapped_file"
@@ -0,0 +1,3 @@
+module SimpleMmap
+ VERSION = '1.0.0'
+end
@@ -0,0 +1,27 @@
+require "test/unit"
+require "tempfile"
+
+require File.dirname(__FILE__) + "/../lib/simple_mmap"
+
+class TestMappedFile < Test::Unit::TestCase
+ def setup
+ @file = Tempfile.new("TestMappedFile.data")
+ File.open(@file.path, "w"){|f| f.puts(('a'..'z').to_a.join) }
+ @map = SimpleMmap::MappedFile.new(@file.path)
+ end
+
+ def teardown
+ @map.close
+ end
+
+ def test_read_window_data
+ assert_equal "abc", @map.read_window_data(0, 3)
+ assert_equal "bc", @map.read_window_data(1, 2)
+ assert_equal "klmno", @map.read_window_data(10, 5)
+ end
+
+ def test_read_window_beyond_buffer_size
+ assert_equal "z\n", @map.read_window_data(25, 5)
+ assert_nil @map.read_window_data(28, 5)
+ end
+end

0 comments on commit 3baee14

Please sign in to comment.