Skip to content

Commit

Permalink
serialize
Browse files Browse the repository at this point in the history
  • Loading branch information
quek committed Jun 3, 2012
1 parent dbb3a8a commit 1cc8e7d
Show file tree
Hide file tree
Showing 5 changed files with 327 additions and 158 deletions.
66 changes: 47 additions & 19 deletions heap.lisp
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
:test 'equalp)
(defconstant +heap-header-length+ 8)
(defparameter *mmap-size* (* 1024 1024))
(defconstant +min-block-size+ 16 "16 byte")
(defconstant +min-block-size+ 8 "8 byte")
(defconstant +block-meta-data-size+ 1 "1 byte. 1 bit 目が +heap-unuse+ の時は未使用")


Expand All @@ -48,11 +48,36 @@
((directory)
(heaps)))

(defconstant +size-of-address+ 7)

(defstruct address
"シリアライズするために 16 byte のアドレス表現を使う。"
"シリアライズするために 7 byte のアドレス表現を使う。"
(segment 0 :type (unsigned-byte 8))
(offset 0 :type (unsigned-byte 120)))
(offset 0 :type (unsigned-byte 56)))


(defmethod serialize ((self address) stream)
(write-byte +tag-address+ stream)
(write-byte (address-segment self) stream)
(write-integer (address-offset self) (1- +size-of-address+) stream))

(defmethod deserialize-by-tag ((tag (eql +tag-address+)) stream)
(make-address :segment (read-byte stream)
:offset (read-integer stream (1- +size-of-address+))))

(defun read-address (stream position)
(let ((buffer (make-buffer +size-of-address+)))
(read-seq-at stream buffer position)
(make-address :segment (svref buffer 0)
:offset (from-bytes buffer 1 #.(1- +size-of-address+)))))

(defun write-address (address stream position)
(let ((buffer (make-buffer +size-of-address+)))
(setf (svref buffer 0) (address-segment address))
(bytes-into buffer 1 (address-offset address) #.(1- +size-of-address+))
(write-seq-at stream buffer position)
address))


(defclass* memory ()
((address)
Expand Down Expand Up @@ -169,10 +194,8 @@
(defmethod heap-file-open :after ((heap-file heap-file))
(heap-file-collect-free-memories heap-file))

(defconstant +heap-use+ 1)
(defconstant +heap-unuse+ 0)
(defconstant +heap-locked+ 1)
(defconstant +heap-unlocked+ 0)
(defconstant +heap-use-bit+ #x01)
(defconstant +heap-lock-bit+ #x02)

(defmethod heap-file-collect-free-memories ((heap-file heap-file))
"しっぽがフラグ"
Expand All @@ -184,23 +207,22 @@
then (+ position offset)
while (< position stream-length)
for byte = (read-byte-at stream position)
if (= +heap-unuse+ (logand 1 byte))
unless (zerop (logand +heap-use-bit+ byte))
collect position)))))

(defmethod heap-file-alloc ((heap-file heap-file))
(with-slots (free-memories stream block-size element-type) heap-file
(with-cas-lock (heap-file)
(if free-memories
(pop free-memories)
(let ((position (stream-length stream)))
(write-byte-at stream (+ position block-size) +heap-use+)
position)))))
(let ((position (or (pop free-memories)
(stream-length stream))))
(write-byte-at stream (+ position block-size) +heap-use-bit+)
position))))

(defmethod heap-file-free ((heap-file heap-file) position)
(with-slots (free-memories stream) heap-file
(with-cas-lock (heap-file)
(push position free-memories)
(write-byte-at stream (+ position (block-size-of heap-file)) +heap-unuse+))))
(write-byte-at stream (+ position (block-size-of heap-file)) 0))))

(defmethod heap-file-read ((heap-file heap-file) position buffer)
(read-seq-at (stream-of heap-file) buffer position
Expand All @@ -218,8 +240,8 @@
(loop
(with-cas-lock (heap-file)
(let ((byte (read-byte-at stream position)))
(when (zerop (logand +heap-locked+ byte))
(write-byte-at stream position (logior +heap-locked+ byte))
(when (zerop (logand +heap-lock-bit+ byte))
(write-byte-at stream position (logior +heap-lock-bit+ byte))
(return-from heap-file-lock))))
(sb-thread:thread-yield)))))

Expand All @@ -228,7 +250,7 @@
(let ((position (+ position block-size)))
(with-cas-lock (heap-file)
(let ((byte (read-byte-at stream position)))
(write-byte-at stream position (logxor +heap-locked+ byte)))))))
(write-byte-at stream position (logxor +heap-lock-bit+ byte)))))))


(let* ((dir (print "/tmp/heap-test/"))
Expand All @@ -244,7 +266,13 @@
(assert (= 0 (address-segment (address-of memory2))))
(assert (= 1 (address-offset (address-of memory2))))
(heap-free heap memory1)
(let ((memory3 (heap-alloc heap 16)))
(let ((memory3 (heap-alloc heap 8)))
(assert (= 0 (address-segment (address-of memory3))))
(assert (= 0 (address-offset (address-of memory3)))))))
(assert (= 0 (address-offset (address-of memory3)))))
(let ((memory4 (heap-alloc heap 8)))
(assert (= 0 (address-segment (address-of memory4))))
(assert (= 2 (address-offset (address-of memory4)))))
(setf memory1 (heap-alloc heap (ash +min-block-size+ 20)))
(assert (= 20 (address-segment (address-of memory1))))
(assert (= 0 (address-offset (address-of memory1))))))
(heap-close heap)))
7 changes: 5 additions & 2 deletions nunumo.asd
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,12 @@
:components ((:file "package")
(:file "type")
(:file "util")
(:file "serialize")
(:file "thread")
(:file "thread-pool")
(:file "mmap")
(:file "heap")
(:file "byte-heap-file")
(:file "thread")
(:file "thread-pool")
(:file "server")
(:file "client")
(:file "nunumo")
Expand All @@ -23,5 +24,7 @@
:hu.dwim.defclass-star
:sb-concurrency
:usocket
:md5
:flexi-streams
:info.read-eval-print.series-ext))

155 changes: 145 additions & 10 deletions serialize.lisp
Original file line number Diff line number Diff line change
@@ -1,20 +1,155 @@
(in-package :nunumo)

(defgeneric serialize (object stream))
(defgeneric deserializing (stream))
(defgeneric deserialize (stream))
(defgeneric deserialize-by-tag (tag stream))


(macrolet ((m (&rest syms)
(let ((value -1))
`(progn
,@(mapcar (lambda (x)
`(defconstant ,x ,(incf value)))
(if (numberp x)
(setf value (1- x))
`(defconstant ,x ,(incf value))))
syms)))))
(m +invalid+
+ignore+
+t+
+nil+
+-1+
+0+
+1+
+positive-integer-8+))
(m +tag-ignore+
+tag-invalid+
+tag-t+
+tag-nil+
+tag--1+
+tag-0+
+tag-1+
#x10
+tag-positive-integer-8+
+tag-positive-integer-16+
+tag-positive-integer-32+
+tag-positive-integer-64+
+tag-positive-integer+
#x20
+tag-negative-integer-8+
+tag-negative-integer-16+
+tag-negative-integer-32+
+tag-negative-integer-64+
+tag-negative-integer+
#x30
+tag-string+
#xe0
+tag-address+
+tag-node+))

(defmethod serialize ((self (eql t)) stream)
(write-byte +tag-t+ stream))

(defmethod serialize ((self (eql nil)) stream)
(write-byte +tag-nil+ stream))

(defmethod serialize ((self (eql -1)) stream)
(write-byte +tag--1+ stream))

(defmethod serialize ((self (eql 0)) stream)
(write-byte +tag-0+ stream))

(defmethod serialize ((self (eql 1)) stream)
(write-byte +tag-1+ stream))

(defmethod serialize ((self integer) stream)
(let* ((plusp (plusp self))
(abs (abs self))
(size (ceiling (integer-length abs) 8)))
(write-byte
(case size
(1 (if plusp +tag-positive-integer-8+ +tag-negative-integer-8+))
(2 (if plusp +tag-positive-integer-16+ +tag-negative-integer-16+))
((3 4) (if plusp +tag-positive-integer-32+ +tag-negative-integer-32+))
((5 6 7 8) (if plusp +tag-positive-integer-64+ +tag-negative-integer-64+))
(t (if plusp +tag-positive-integer+ +tag-negative-integer+)))
stream)
(when (< 8 size)
(serialize size stream))
(write-integer abs size stream)))

(defmethod serialize ((self string) stream)
(write-byte +tag-string+ stream)
(let* ((octets (flex:string-to-octets self :external-format :utf8))
(length (length octets)))
(serialize length stream)
(write-sequence octets stream)))


(defmethod deserialize (stream)
(let ((tag (read-byte stream)))
(deserialize-by-tag tag stream)))

(defmethod deserialize-by-tag ((tag (eql +tag-t+)) stream)
t)

(defmethod deserialize-by-tag ((tag (eql +tag-nil+)) stream)
nil)

(defmethod deserialize-by-tag ((tag (eql +tag--1+)) stream)
-1)

(defmethod deserialize-by-tag ((tag (eql +tag-0+)) stream)
0)

(defmethod deserialize-by-tag ((tag (eql +tag-1+)) stream)
1)

(defmethod deserialize-by-tag ((tag (eql +tag-positive-integer-8+)) stream)
(read-byte stream))

(defmethod deserialize-by-tag ((tag (eql +tag-positive-integer-16+)) stream)
(read-integer stream 2))

(defmethod deserialize-by-tag ((tag (eql +tag-positive-integer-32+)) stream)
(read-integer stream 4))

(defmethod deserialize-by-tag ((tag (eql +tag-positive-integer-64+)) stream)
(read-integer stream 8))

(defmethod deserialize-by-tag ((tag (eql +tag-positive-integer+)) stream)
(let ((size (deserialize stream)))
(read-integer stream size)))

(defmethod deserialize-by-tag ((tag (eql +tag-negative-integer-8+)) stream)
(- (read-byte stream)))

(defmethod deserialize-by-tag ((tag (eql +tag-negative-integer-16+)) stream)
(- (read-integer stream 2)))

(defmethod deserialize-by-tag ((tag (eql +tag-negative-integer-32+)) stream)
(- (read-integer stream 4)))

(defmethod deserialize-by-tag ((tag (eql +tag-negative-integer-64+)) stream)
(- (read-integer stream 8)))

(defmethod deserialize-by-tag ((tag (eql +tag-negative-integer+)) stream)
(let ((size (deserialize stream)))
(- (read-integer stream size))))

(defmethod deserialize-by-tag ((tag (eql +tag-string+)) stream)
(let* ((length (deserialize stream))
(buffer (make-buffer length)))
(read-sequence buffer stream)
(flex:octets-to-string buffer :external-format :utf8)))



(iterate ((x (scan '(t nil -1 0 1
255
65535
4294967295
18446744073709551615
12345678901234567890123456789012345678901234567890
-255
-65535
-4294967295
-18446744073709551615
-12345678901234567890123456789012345678901234567890
"hello"))))
(flex:with-input-from-sequence
(in (flex:with-output-to-sequence (out)
(serialize x out)))
(assert (equal x (deserialize in)))))

Loading

0 comments on commit 1cc8e7d

Please sign in to comment.