strokemap: optimize save_ora, get rid of protobuf library

Most of the saving time was spent inside protobuf while saving the
strokemap.  We now use a different binary format. Old strokemaps
in existing files will be discarded.

In addition, about 50% of the strokemap size and saving time
was used for the brush settings. Those were saved as string with
each stroke. We now save only an index in a separate brush table.

.gitignore

@@ -16,6 +16,5 @@ options.cache
 *.mo
 *.pot
 POTFILES.in
-lib/strokemap_pb2.py
 .#*
 *.pstats

README

@@ -13,8 +13,8 @@ Building on Linux:
     Install:
     scons prefix=/usr/local install
 
-    Required: pygtk, python, swig, gtk, numpy, pycairo(>=1.4), protobuf, libpng
-    Debian users: apt-get install g++ python-dev libglib2.0-dev python-numpy swig scons gettext python-protobuf protobuf-compiler libpng12-dev
+    Required: pygtk, python, swig, gtk, numpy, pycairo(>=1.4), libpng
+    Debian users: apt-get install g++ python-dev libglib2.0-dev python-numpy swig scons gettext libpng12-dev
 
     Recommended: a pressure sensitive input device (graphic tablet)
 

lib/SConscript

@@ -19,6 +19,4 @@ elif sys.platform == "darwin":
 else:
     module = env.LoadableModule('_mypaintlib', Split(src), SHLIBPREFIX="")
 
-env.Command('strokemap_pb2.py', 'strokemap.proto', 'protoc --python_out=. lib/strokemap.proto')
-
 Return('module')

lib/document.py

@@ -8,6 +8,7 @@
 
 import os, zipfile, tempfile, time, traceback
 join = os.path.join
+from cStringIO import StringIO
 import xml.etree.ElementTree as ET
 from gtk import gdk
 import gobject, numpy
@@ -391,9 +392,11 @@ def add_layer(x, y, opac, surface, name, layer_name, visible=True, rect=[]):
             x, y, w, h = l.surface.get_bbox()
             el = add_layer(x-x0, y-y0, opac, l.surface, 'data/layer%03d.png' % idx, l.name, l.visible, rect=(x, y, w, h))
             # strokemap
-            data = l.save_strokemap_to_string(-x, -y)
+            sio = StringIO()
+            l.save_strokemap_to_file(sio, -x, -y)
+            data = sio.getvalue(); sio.close()
             name = 'data/layer%03d_strokemap.dat' % idx
-            el.attrib['mypaint_strokemap'] = name
+            el.attrib['mypaint_strokemap_v2'] = name
             write_file_str(name, data)
 
         # save background as layer (solid color or tiled)
@@ -513,13 +516,14 @@ def get_layers_list(root, x=0,y=0):
             self.set_layer_visibility(visible, layer)
             print '  %.3fs converting pixbuf to layer format' % (time.time() - t1)
             # strokemap
-            fname = a.get('mypaint_strokemap', None)
+            fname = a.get('mypaint_strokemap_v2', None)
             if fname:
                 if x % N or y % N:
                     print 'Warning: dropping non-aligned strokemap'
                 else:
-                    data = z.read(fname)
-                    layer.load_strokemap_from_string(data, x, y)
+                    sio = StringIO(z.read(fname))
+                    layer.load_strokemap_from_file(sio, x, y)
+                    sio.close()
 
         os.rmdir(tempdir)
 

lib/layer.py

@@ -6,9 +6,10 @@
 # the Free Software Foundation; either version 2 of the License, or
 # (at your option) any later version.
 
+import struct, zlib
 from numpy import *
 
-import tiledsurface, strokemap, strokemap_pb2
+import tiledsurface, strokemap
 
 class Layer:
     def __init__(self,name=""):
@@ -26,7 +27,7 @@ def get_effective_opacity(self):
     effective_opacity = property(get_effective_opacity)
 
     def clear(self):
-        self.strokes = [] # contains StrokeInfo instances (not stroke.Stroke)
+        self.strokes = [] # contains StrokeShape instances (not stroke.Stroke)
         self.surface.clear()
 
     def load_from_pixbuf(self, pixbuf):
@@ -44,24 +45,52 @@ def load_snapshot(self, data):
     def add_stroke(self, stroke, snapshot_before):
         before = snapshot_before[1] # extract surface snapshot
         after  = self.surface.save_snapshot()
-        info = strokemap.StrokeInfo()
-        info.init_from_snapshots(stroke.brush_settings, before, after)
-        self.strokes.append(info)
+        shape = strokemap.StrokeShape()
+        shape.init_from_snapshots(before, after)
+        shape.brush_string = stroke.brush_settings
+        self.strokes.append(shape)
 
-    def save_strokemap_to_string(self, translate_x, translate_y):
-        sl = strokemap_pb2.StrokeList()
+    def save_strokemap_to_file(self, f, translate_x, translate_y):
+        brush2id = {}
         for stroke in self.strokes:
-            stroke_pb = sl.strokes.add()
-            stroke.save_to_pb(stroke_pb, translate_x, translate_y)
-        return sl.SerializeToString()
-
-    def load_strokemap_from_string(self, data, translate_x, translate_y):
-        sl = strokemap_pb2.StrokeList()
-        sl.ParseFromString(data)
-        for stroke_pb in sl.strokes:
-            stroke = strokemap.StrokeInfo()
-            stroke.init_from_pb(stroke_pb, translate_x, translate_y)
-            self.strokes.append(stroke)
+            s = stroke.brush_string
+            # save brush (if not already known)
+            if s not in brush2id:
+                brush2id[s] = len(brush2id)
+                s = zlib.compress(s)
+                f.write('b')
+                f.write(struct.pack('>I', len(s)))
+                f.write(s)
+            # save stroke
+            s = stroke.save_to_string(translate_x, translate_y)
+            f.write('s')
+            f.write(struct.pack('>II', brush2id[stroke.brush_string], len(s)))
+            f.write(s)
+        f.write('}')
+
+
+    def load_strokemap_from_file(self, f, translate_x, translate_y):
+        assert not self.strokes
+        brushes = []
+        while True:
+            t = f.read(1)
+            if t == 'b':
+                length, = struct.unpack('>I', f.read(4))
+                tmp = f.read(length)
+                brushes.append(zlib.decompress(tmp))
+                print 'b', len(tmp)
+            elif t == 's':
+                brush_id, length = struct.unpack('>II', f.read(2*4))
+                stroke = strokemap.StrokeShape()
+                tmp = f.read(length)
+                stroke.init_from_string(tmp, translate_x, translate_y)
+                stroke.brush_string = brushes[brush_id]
+                self.strokes.append(stroke)
+                print 's', len(tmp)
+            elif t == '}':
+                break
+            else:
+                assert False, 'invalid strokemap'
 
     def merge_into(self, dst):
         """

lib/strokemap.py

@@ -6,28 +6,26 @@
 # the Free Software Foundation; either version 2 of the License, or
 # (at your option) any later version.
 
-import time
+import time, struct
 import zlib
 from numpy import *
 
-import tiledsurface, idletask, strokemap_pb2
+import tiledsurface, idletask
 N = tiledsurface.N
 
 tasks = idletask.Processor(max_pending=6)
 
-class StrokeInfo:
+class StrokeShape:
     """
-    This class stores permanent (saved with image) information about a
-    single stroke. Mainly this is the stroke shape and the brush
-    settings that were used. Needed to pick brush from canvas.
+    This class stores the shape of a stroke in as a 1-bit bitmap. The
+    information is stored in compressed memory blocks of the size of a
+    tile (for fast lookup).
     """
     def __init__(self):
         self.strokemap = {}
-        self.brush = None
 
-    def init_from_snapshots(self, brush_string, snapshot_before, snapshot_after):
+    def init_from_snapshots(self, snapshot_before, snapshot_after):
         assert not self.strokemap
-        self.brush_string = brush_string
         # extract the layer from each snapshot
         a, b = snapshot_before.tiledict, snapshot_after.tiledict
         # enumerate all tiles that have changed
@@ -66,27 +64,30 @@ def work(tx=tx, ty=ty):
 
             tasks.add_work(work, weight=1.0/len(tiles_modified))
 
-    def init_from_pb(self, stroke_pb, translate_x, translate_y):
+    def init_from_string(self, data, translate_x, translate_y):
         assert not self.strokemap
         assert translate_x % N == 0
         assert translate_y % N == 0
         translate_x /= N
         translate_y /= N
-        for t in stroke_pb.tiles:
-            self.strokemap[t.tx + translate_x, t.ty + translate_y] = t.data_compressed
-        self.brush_string = zlib.decompress(stroke_pb.brush_string_compressed)
+        while data:
+            tx, ty, size = struct.unpack('>iiI', data[:3*4])
+            compressed_bitmap = data[3*4:size+3*4]
+            self.strokemap[tx + translate_x, ty + translate_y] = compressed_bitmap
+            data = data[size+3*4:]
 
-    def save_to_pb(self, stroke_pb, translate_x, translate_y):
+    def save_to_string(self, translate_x, translate_y):
         assert translate_x % N == 0
         assert translate_y % N == 0
         translate_x /= N
         translate_y /= N
         tasks.finish_all()
-        for (tx, ty), data in self.strokemap.iteritems():
-            t = stroke_pb.tiles.add()
-            t.tx, t.ty = tx + translate_x, ty + translate_y
-            t.data_compressed = data
-        stroke_pb.brush_string_compressed = zlib.compress(self.brush_string)
+        data = ''
+        for (tx, ty), compressed_bitmap in self.strokemap.iteritems():
+            tx, ty = tx + translate_x, ty + translate_y
+            data += struct.pack('>iiI', tx, ty, len(compressed_bitmap))
+            data += compressed_bitmap
+        return data
 
     def touches_pixel(self, x, y):
         tasks.finish_all()