Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Newer
Older
100644 410 lines (337 sloc) 13.11 kb
8184302 @sorear Start draft of serialization/deserialization code
authored
1 using System;
d832c02 @sorear Second draft of serialization code
authored
2 using System.IO;
3 using System.Security.Cryptography;
b25256f @sorear Add Serialize.cs to build, fix build errors
authored
4 using System.Collections.Generic;
d832c02 @sorear Second draft of serialization code
authored
5 using System.Text;
8184302 @sorear Start draft of serialization/deserialization code
authored
6
cdde7e0 @sorear Notes on four kinds of module scope
authored
7 // Here in Niecza we have four different kinds of unit scopes:
8 //
9 // * COMPILING::UNIT, aka RuntimeUnit: one of these exists for every
10 // call into the compiler, whether eval or module. The REPL will
11 // be considered as if it were eval.
12 //
13 // * Serialization scopes, which are created when compiling modules
14 // or when pre-compiling a main program. During execution there is
15 // no current serialization scope; evals inherit the serialization
16 // scope or lack thereof that was in effect.
17 //
18 // * Assembly scopes, which are an artifact of the CLR and almost align
19 // with precompilation scopes, except that they have to exist always
20 // because methods cannot be created free-floating in CLR 2.x.
21 //
22 // An assembly scope is created for all serialization scopes, and
23 // non-saved anonymous assemblies are created for eval-and-run of
24 // a file and non-BEGIN-time evals.
25 //
26 // * GLOBAL scope is very much like serialization scope except that there
27 // is a "true globals" scope that is used when not serializing.
28
8184302 @sorear Start draft of serialization/deserialization code
authored
29 // This implements "bounded serialization" for Niecza. Unfortunately
30 // the CLR's builtin serialization can't efficiently be made bounded,
31 // and anyway it would be nice if the serialization format could be
32 // transported across backends.
33
34 // TODO: implement a more Storable-like interface.
d832c02 @sorear Second draft of serialization code
authored
35
36 // Note, the serialization subsystem is *NOT* thread safe !
8184302 @sorear Start draft of serialization/deserialization code
authored
37 namespace Niecza.Serialization {
d832c02 @sorear Second draft of serialization code
authored
38 // Information kept on a serialization unit after loading or storing,
39 // but not before storing.
40 class SerUnit {
41 internal string name; // eg "File.Copy"
42 internal byte[] hash; // hash of entire file, filled at write time
43 internal object[] bynum; // objects in unit
44 internal object root; // the RuntimeUnit object
45 internal int nobj;
46 }
8184302 @sorear Start draft of serialization/deserialization code
authored
47
48 // The central feature of *bounded* serialization is that object
49 // registries are kept distinct from the (de)serializer, and can
50 // be shared between serialization runs.
51 class ObjectRegistry {
d832c02 @sorear Second draft of serialization code
authored
52 // TODO: investigate a more specialized representation,
53 // ideally not having to hash as many objects
54 struct ObjRef {
b25256f @sorear Add Serialize.cs to build, fix build errors
authored
55 public SerUnit unit;
56 public int id;
8184302 @sorear Start draft of serialization/deserialization code
authored
57 }
d832c02 @sorear Second draft of serialization code
authored
58 Dictionary<object,ObjRef> byref = new Dictionary<object,ObjRef>();
8184302 @sorear Start draft of serialization/deserialization code
authored
59
d832c02 @sorear Second draft of serialization code
authored
60 Dictionary<string,SerUnit> units =
61 new Dictionary<string,SerUnit>();
62
63 static readonly HashAlgorithm hash = SHA256.Create();
64 static readonly string signature = "Niecza-Serialized-Module";
65 static readonly int version = 1;
66
67 // Routines for use by serialization code
68 public bool CheckWriteObject(SerUnit into, object o,
69 out SerUnit lui, out int id) {
70 ObjRef or;
71 if (byref.TryGetValue(o, out or)) {
72 lui = or.unit;
73 id = or.id;
8184302 @sorear Start draft of serialization/deserialization code
authored
74 return true;
d832c02 @sorear Second draft of serialization code
authored
75 }
8184302 @sorear Start draft of serialization/deserialization code
authored
76
d832c02 @sorear Second draft of serialization code
authored
77 if (into.nobj == into.bynum.Length)
78 Array.Resize(ref into.bynum, into.nobj * 2);
8184302 @sorear Start draft of serialization/deserialization code
authored
79
b25256f @sorear Add Serialize.cs to build, fix build errors
authored
80 or.unit = lui = into;
d832c02 @sorear Second draft of serialization code
authored
81 id = or.id = into.nobj++;
82 into.bynum[id] = o;
83
84 byref[o] = or;
8184302 @sorear Start draft of serialization/deserialization code
authored
85
86 return false;
87 }
d832c02 @sorear Second draft of serialization code
authored
88
89 // Routines for use by compilation manager
90
91 // Loads a single unit from the compiled-data directory.
92 // Will throw a ThawException if a stale reference is encountered
93 // or other data format error.
94 public SerUnit LoadUnit(string name) {
95 SerUnit su;
96
97 // is the unit already loaded?
98 if (units.TryGetValue(name, out su))
99 return su;
100
101 string file = Path.Combine(AppDomain.CurrentDomain.BaseDirectory,
102 name + ".ser");
103 byte[] bytes = File.ReadAllBytes(file);
104
105 su = new SerUnit();
106 su.name = name;
107 su.hash = hash.ComputeHash(bytes);
108
b25256f @sorear Add Serialize.cs to build, fix build errors
authored
109 ThawBuffer tb = new ThawBuffer(this, su, bytes);
d832c02 @sorear Second draft of serialization code
authored
110
111 bool success = false;
112 try {
113 string rsig = tb.String();
114 if (rsig != signature)
115 throw new ThawException("signature mismatch loading " + file);
116 int rver = tb.Int();
117 if (rver != version)
118 throw new ThawException("version mismatch loading " + file);
119
b25256f @sorear Add Serialize.cs to build, fix build errors
authored
120 su.root = tb.ObjRef();
d832c02 @sorear Second draft of serialization code
authored
121 success = true;
122 } finally {
123 // don't leave half-read units in the map
124 if (!success)
125 UnloadUnit(name);
126 }
127
128 return su;
129 }
130
131 // removes a stale unit so a new version can be saved over it.
132 public void UnloadUnit(string name) {
133 SerUnit su = units[name];
134 units.Remove(name);
135
136 for (int i = 0; i < su.nobj; i++)
137 byref.Remove(su.bynum[i]);
138 }
139
140 public SerUnit SaveUnit(string name, IFreeze root) {
141 SerUnit su = new SerUnit();
142 su.name = name;
143 su.root = root;
144
145 if (units.ContainsKey(name))
b25256f @sorear Add Serialize.cs to build, fix build errors
authored
146 throw new InvalidOperationException("unit " +name+ " exists");
d832c02 @sorear Second draft of serialization code
authored
147
148 bool success = false;
149 string file = Path.Combine(AppDomain.CurrentDomain.BaseDirectory,
150 name + ".ser");
151
152 FreezeBuffer fb = new FreezeBuffer(this, su);
153
154 try {
155 fb.String(signature);
156 fb.Int(version);
157 fb.ObjRef(root);
158
159 byte[] data = fb.GetData();
160 su.hash = hash.ComputeHash(data);
161 File.WriteAllBytes(file, data);
162 success = true;
163 } finally {
164 if (!success)
165 UnloadUnit(name);
166 }
167
168 return su;
169 }
8184302 @sorear Start draft of serialization/deserialization code
authored
170 }
171
172 // One of these codes is written at the beginning of every object ref
173 enum SerializationCode {
d832c02 @sorear Second draft of serialization code
authored
174 // special
175 Null,
176
8184302 @sorear Start draft of serialization/deserialization code
authored
177 // existing objects
178 ForeignRef,
179 SelfRef,
180 NewUnitRef,
181 }
182
183 // An instance of this class is used to serialize serialization units
184 class FreezeBuffer {
185 byte[] data;
186 int wpointer;
187
d832c02 @sorear Second draft of serialization code
authored
188 Dictionary<SerUnit,int> unit_to_offset;
189 int usedunits;
8184302 @sorear Start draft of serialization/deserialization code
authored
190
191 ObjectRegistry reg;
d832c02 @sorear Second draft of serialization code
authored
192 SerUnit unit;
8184302 @sorear Start draft of serialization/deserialization code
authored
193
d832c02 @sorear Second draft of serialization code
authored
194 internal FreezeBuffer(ObjectRegistry reg, SerUnit unit) {
8184302 @sorear Start draft of serialization/deserialization code
authored
195 this.reg = reg;
d832c02 @sorear Second draft of serialization code
authored
196 this.unit = unit;
197 unit_to_offset = new Dictionary<SerUnit,int>();
8184302 @sorear Start draft of serialization/deserialization code
authored
198 data = new byte[256];
199 }
200
b25256f @sorear Add Serialize.cs to build, fix build errors
authored
201 internal byte[] GetData() {
202 byte[] ret = new byte[wpointer];
203 Array.Copy(data, ret, ret.Length);
204 return ret;
205 }
206
8184302 @sorear Start draft of serialization/deserialization code
authored
207 void Ensure(int ct) {
208 while (ct + wpointer > data.Length)
209 Array.Resize(ref data, data.Length * 2);
210 }
211
212 public void Byte(byte x) {
213 Ensure(1);
214 data[wpointer++] = x;
215 }
216
217 public void Short(short x) {
218 Ensure(2);
219 data[wpointer++] = (byte)(x >> 8);
220 data[wpointer++] = (byte)(x );
221 }
222
223 public void Int(int x) {
224 Ensure(4);
225 data[wpointer++] = (byte)(x >> 24);
226 data[wpointer++] = (byte)(x >> 16);
227 data[wpointer++] = (byte)(x >> 8);
228 data[wpointer++] = (byte)(x );
229 }
230
231 public void Long(long x) {
232 Ensure(8);
233 data[wpointer++] = (byte)(x >> 56);
234 data[wpointer++] = (byte)(x >> 48);
235 data[wpointer++] = (byte)(x >> 40);
236 data[wpointer++] = (byte)(x >> 32);
237 data[wpointer++] = (byte)(x >> 24);
238 data[wpointer++] = (byte)(x >> 16);
239 data[wpointer++] = (byte)(x >> 8);
240 data[wpointer++] = (byte)(x );
241 }
242
243 public void String(string s) {
244 if (s == null) {
245 Int(-1);
246 } else {
247 Int(s.Length);
248 foreach (char ch in s)
249 Short((short)ch);
250 }
251 }
252
d832c02 @sorear Second draft of serialization code
authored
253 // This is the main routine you should call from your Freeze
254 // callbacks to freeze an object
8184302 @sorear Start draft of serialization/deserialization code
authored
255 public void ObjRef(IFreeze o) {
d832c02 @sorear Second draft of serialization code
authored
256 int id;
257 SerUnit altunit;
258 if (o == null) { // null pointers are special
259 Byte((byte)SerializationCode.Null);
260 return;
261 }
262
263 if (reg.CheckWriteObject(unit, o, out altunit, out id)) {
8184302 @sorear Start draft of serialization/deserialization code
authored
264 if (altunit == unit) {
265 Byte((byte)SerializationCode.SelfRef);
266 } else {
d832c02 @sorear Second draft of serialization code
authored
267 int altcode;
268 if (!unit_to_offset.TryGetValue(altunit, out altcode)) {
8184302 @sorear Start draft of serialization/deserialization code
authored
269 Byte((byte)SerializationCode.NewUnitRef);
b25256f @sorear Add Serialize.cs to build, fix build errors
authored
270 String(altunit.name);
d832c02 @sorear Second draft of serialization code
authored
271 // save the hash too so stale refs can be caught
272 foreach (byte b in altunit.hash) Byte(b);
273
8184302 @sorear Start draft of serialization/deserialization code
authored
274 unit_to_offset[altunit] = usedunits++;
275 } else {
276 Byte((byte)SerializationCode.ForeignRef);
d832c02 @sorear Second draft of serialization code
authored
277 Int(altcode);
8184302 @sorear Start draft of serialization/deserialization code
authored
278 }
279 }
280 Int((int)id);
281 } else {
d832c02 @sorear Second draft of serialization code
authored
282 // must take responsibility for saving the tag
8184302 @sorear Start draft of serialization/deserialization code
authored
283 o.Freeze(this);
284 }
285 }
286 }
287
288 // Note that this interface only handles freezing - thaw is done using
289 // a switch statement.
290 interface IFreeze {
291 void Freeze(FreezeBuffer fb);
292 }
293
294 class ThawBuffer {
295 byte[] data;
296 int rpointer;
297 ObjectRegistry reg;
298
d832c02 @sorear Second draft of serialization code
authored
299 SerUnit[] unit_map = new SerUnit[8];
8184302 @sorear Start draft of serialization/deserialization code
authored
300 int refed_units;
d832c02 @sorear Second draft of serialization code
authored
301 SerUnit unit;
8184302 @sorear Start draft of serialization/deserialization code
authored
302
b25256f @sorear Add Serialize.cs to build, fix build errors
authored
303 internal ThawBuffer(ObjectRegistry reg, SerUnit unit, byte[] data) {
8184302 @sorear Start draft of serialization/deserialization code
authored
304 this.data = data;
305 this.reg = reg;
d832c02 @sorear Second draft of serialization code
authored
306 this.unit = unit;
8184302 @sorear Start draft of serialization/deserialization code
authored
307 }
308
b25256f @sorear Add Serialize.cs to build, fix build errors
authored
309 public byte Byte() { return data[rpointer++]; }
310
311 public short Short() {
312 return (short)((((int)Byte()) << 8) | Byte());
313 }
314
315 public int Int() {
316 return (((int)Byte()) << 24) | (((int)Byte()) << 16) |
317 (((int)Byte()) << 8) | ((int)Byte());
318 }
319
320 public long Long() {
321 // try to do as much as possible in 32-bit precision,
322 // but suppress sign extension
323 return (((long)Int()) << 32) | (long)(uint)Int();
324 }
325
326 public string String() {
327 int l = Int();
328
329 if (l < 0) return null;
330 char[] cb = new char[l];
331
332 for (int i = 0; i < l; i++)
333 cb[i] = (char)Short();
334
335 return new string(cb);
336 }
337
338 public byte[] Bytes(int k) {
339 byte[] buf = new byte[k];
340
341 for (int i = 0; i < k; i++)
342 buf[i] = Byte();
343
344 return buf;
345 }
346
347 public object ObjRef() {
8184302 @sorear Start draft of serialization/deserialization code
authored
348 var tag = (SerializationCode)Byte();
349 int i, j;
350 switch(tag) {
d832c02 @sorear Second draft of serialization code
authored
351 case SerializationCode.Null:
352 return null;
8184302 @sorear Start draft of serialization/deserialization code
authored
353 case SerializationCode.SelfRef:
354 i = Int();
d832c02 @sorear Second draft of serialization code
authored
355 return unit.bynum[i];
8184302 @sorear Start draft of serialization/deserialization code
authored
356 case SerializationCode.ForeignRef:
357 i = Int();
358 j = Int();
d832c02 @sorear Second draft of serialization code
authored
359 return unit_map[i].bynum[j];
8184302 @sorear Start draft of serialization/deserialization code
authored
360 case SerializationCode.NewUnitRef:
d832c02 @sorear Second draft of serialization code
authored
361 return LoadNewUnit();
8184302 @sorear Start draft of serialization/deserialization code
authored
362 default:
363 throw new ThawException("unexpected object tag" + (byte)tag);
364 }
365 }
d832c02 @sorear Second draft of serialization code
authored
366
367 object LoadNewUnit() {
368 string name = String();
369 if (refed_units == unit_map.Length)
370 Array.Resize(ref unit_map, refed_units * 2);
371
372 SerUnit su = reg.LoadUnit(name);
373 unit_map[refed_units++] = su;
374
375 byte[] hash = Bytes(su.hash.Length);
376
377 for (int i = 0; i < hash.Length; i++)
378 if (hash[i] != su.hash[i])
379 goto badhash;
380
381 int ix = Int();
382 return su.bynum[ix];
383
384 badhash:
385 StringBuilder sb = new StringBuilder();
386 sb.AppendFormat("Hash mismatch for unit {0} referenced from {1}",
387 su.name, unit.name);
388
389 sb.Append(", wanted ");
390 foreach (byte b in hash)
391 sb.AppendFormat("{0:X2}", b);
392
393 sb.Append(", got ");
394 foreach (byte b in su.hash)
395 sb.AppendFormat("{0:X2}", b);
396
397 throw new ThawException(sb.ToString());
398 }
399 }
400
401 // Thrown to indicate data format problems in the serialized stream
402 // Not necessarily bugs; could also indicate stale files, including
403 // cases where the data format is changed and cases where a depended
404 // file was recreated
405 class ThawException : Exception {
406 public ThawException(string s) : base(s) { }
407 public ThawException() : base() { }
8184302 @sorear Start draft of serialization/deserialization code
authored
408 }
409 }
Something went wrong with that request. Please try again.