Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Newer
Older
100644 545 lines (458 sloc) 16.622 kb
8184302 @sorear Start draft of serialization/deserialization code
authored
1 using System;
d832c02 @sorear Second draft of serialization code
authored
2 using System.IO;
3 using System.Security.Cryptography;
b25256f @sorear Add Serialize.cs to build, fix build errors
authored
4 using System.Collections.Generic;
d832c02 @sorear Second draft of serialization code
authored
5 using System.Text;
8184302 @sorear Start draft of serialization/deserialization code
authored
6
cdde7e0 @sorear Notes on four kinds of module scope
authored
7 // Here in Niecza we have four different kinds of unit scopes:
8 //
9 // * COMPILING::UNIT, aka RuntimeUnit: one of these exists for every
10 // call into the compiler, whether eval or module. The REPL will
11 // be considered as if it were eval.
12 //
13 // * Serialization scopes, which are created when compiling modules
14 // or when pre-compiling a main program. During execution there is
15 // no current serialization scope; evals inherit the serialization
16 // scope or lack thereof that was in effect.
17 //
18 // * Assembly scopes, which are an artifact of the CLR and almost align
19 // with precompilation scopes, except that they have to exist always
20 // because methods cannot be created free-floating in CLR 2.x.
21 //
22 // An assembly scope is created for all serialization scopes, and
23 // non-saved anonymous assemblies are created for eval-and-run of
24 // a file and non-BEGIN-time evals.
25 //
26 // * GLOBAL scope is very much like serialization scope except that there
27 // is a "true globals" scope that is used when not serializing.
28
8184302 @sorear Start draft of serialization/deserialization code
authored
29 // This implements "bounded serialization" for Niecza. Unfortunately
30 // the CLR's builtin serialization can't efficiently be made bounded,
31 // and anyway it would be nice if the serialization format could be
32 // transported across backends.
33
34 // TODO: implement a more Storable-like interface.
d832c02 @sorear Second draft of serialization code
authored
35
36 // Note, the serialization subsystem is *NOT* thread safe !
8184302 @sorear Start draft of serialization/deserialization code
authored
37 namespace Niecza.Serialization {
d832c02 @sorear Second draft of serialization code
authored
38 // Information kept on a serialization unit after loading or storing,
39 // but not before storing.
40 class SerUnit {
41 internal string name; // eg "File.Copy"
42 internal byte[] hash; // hash of entire file, filled at write time
8ed36ea @sorear Wire up serializer to run after pre-compiler
authored
43 internal object[] bynum = new object[8]; // objects in unit
d832c02 @sorear Second draft of serialization code
authored
44 internal object root; // the RuntimeUnit object
8ed36ea @sorear Wire up serializer to run after pre-compiler
authored
45 internal int nobj; // = 0
d832c02 @sorear Second draft of serialization code
authored
46 }
8184302 @sorear Start draft of serialization/deserialization code
authored
47
48 // The central feature of *bounded* serialization is that object
49 // registries are kept distinct from the (de)serializer, and can
50 // be shared between serialization runs.
51 class ObjectRegistry {
d832c02 @sorear Second draft of serialization code
authored
52 // TODO: investigate a more specialized representation,
53 // ideally not having to hash as many objects
54 struct ObjRef {
b25256f @sorear Add Serialize.cs to build, fix build errors
authored
55 public SerUnit unit;
56 public int id;
8184302 @sorear Start draft of serialization/deserialization code
authored
57 }
d832c02 @sorear Second draft of serialization code
authored
58 Dictionary<object,ObjRef> byref = new Dictionary<object,ObjRef>();
8184302 @sorear Start draft of serialization/deserialization code
authored
59
d832c02 @sorear Second draft of serialization code
authored
60 Dictionary<string,SerUnit> units =
61 new Dictionary<string,SerUnit>();
62
63 static readonly HashAlgorithm hash = SHA256.Create();
64 static readonly string signature = "Niecza-Serialized-Module";
65 static readonly int version = 1;
66
67 // Routines for use by serialization code
68 public bool CheckWriteObject(SerUnit into, object o,
69 out SerUnit lui, out int id) {
70 ObjRef or;
71 if (byref.TryGetValue(o, out or)) {
72 lui = or.unit;
73 id = or.id;
8184302 @sorear Start draft of serialization/deserialization code
authored
74 return true;
d832c02 @sorear Second draft of serialization code
authored
75 }
8184302 @sorear Start draft of serialization/deserialization code
authored
76
d832c02 @sorear Second draft of serialization code
authored
77 if (into.nobj == into.bynum.Length)
78 Array.Resize(ref into.bynum, into.nobj * 2);
8184302 @sorear Start draft of serialization/deserialization code
authored
79
b25256f @sorear Add Serialize.cs to build, fix build errors
authored
80 or.unit = lui = into;
d832c02 @sorear Second draft of serialization code
authored
81 id = or.id = into.nobj++;
82 into.bynum[id] = o;
83
84 byref[o] = or;
8184302 @sorear Start draft of serialization/deserialization code
authored
85
86 return false;
87 }
d832c02 @sorear Second draft of serialization code
authored
88
89 // Routines for use by compilation manager
90
91 // Loads a single unit from the compiled-data directory.
92 // Will throw a ThawException if a stale reference is encountered
93 // or other data format error.
94 public SerUnit LoadUnit(string name) {
95 SerUnit su;
96
97 // is the unit already loaded?
98 if (units.TryGetValue(name, out su))
99 return su;
100
101 string file = Path.Combine(AppDomain.CurrentDomain.BaseDirectory,
102 name + ".ser");
103 byte[] bytes = File.ReadAllBytes(file);
104
105 su = new SerUnit();
106 su.name = name;
107 su.hash = hash.ComputeHash(bytes);
108
b25256f @sorear Add Serialize.cs to build, fix build errors
authored
109 ThawBuffer tb = new ThawBuffer(this, su, bytes);
d832c02 @sorear Second draft of serialization code
authored
110
8ed36ea @sorear Wire up serializer to run after pre-compiler
authored
111 units[name] = su;
d832c02 @sorear Second draft of serialization code
authored
112 bool success = false;
113 try {
114 string rsig = tb.String();
115 if (rsig != signature)
116 throw new ThawException("signature mismatch loading " + file);
117 int rver = tb.Int();
118 if (rver != version)
119 throw new ThawException("version mismatch loading " + file);
120
b25256f @sorear Add Serialize.cs to build, fix build errors
authored
121 su.root = tb.ObjRef();
d832c02 @sorear Second draft of serialization code
authored
122 success = true;
123 } finally {
124 // don't leave half-read units in the map
125 if (!success)
126 UnloadUnit(name);
127 }
128
129 return su;
130 }
131
132 // removes a stale unit so a new version can be saved over it.
133 public void UnloadUnit(string name) {
8ed36ea @sorear Wire up serializer to run after pre-compiler
authored
134 if (!units.ContainsKey(name))
135 return;
d832c02 @sorear Second draft of serialization code
authored
136 SerUnit su = units[name];
137 units.Remove(name);
138
139 for (int i = 0; i < su.nobj; i++)
140 byref.Remove(su.bynum[i]);
141 }
142
143 public SerUnit SaveUnit(string name, IFreeze root) {
144 SerUnit su = new SerUnit();
145 su.name = name;
146 su.root = root;
147
148 if (units.ContainsKey(name))
b25256f @sorear Add Serialize.cs to build, fix build errors
authored
149 throw new InvalidOperationException("unit " +name+ " exists");
d832c02 @sorear Second draft of serialization code
authored
150
151 bool success = false;
152 string file = Path.Combine(AppDomain.CurrentDomain.BaseDirectory,
153 name + ".ser");
154
155 FreezeBuffer fb = new FreezeBuffer(this, su);
156
8ed36ea @sorear Wire up serializer to run after pre-compiler
authored
157 units[name] = su;
158
d832c02 @sorear Second draft of serialization code
authored
159 try {
160 fb.String(signature);
161 fb.Int(version);
162 fb.ObjRef(root);
163
164 byte[] data = fb.GetData();
165 su.hash = hash.ComputeHash(data);
166 File.WriteAllBytes(file, data);
167 success = true;
168 } finally {
169 if (!success)
170 UnloadUnit(name);
171 }
172
173 return su;
174 }
8184302 @sorear Start draft of serialization/deserialization code
authored
175 }
176
177 // One of these codes is written at the beginning of every object ref
1993029 @sorear Start implementing serialization for our objects
authored
178 enum SerializationCode : byte {
d832c02 @sorear Second draft of serialization code
authored
179 // special
180 Null,
181
8184302 @sorear Start draft of serialization/deserialization code
authored
182 // existing objects
183 ForeignRef,
184 SelfRef,
185 NewUnitRef,
1993029 @sorear Start implementing serialization for our objects
authored
186
187 // types of new object
188 RuntimeUnit,
189 SubInfo,
190 STable,
191 StashEnt,
d91ecdf @sorear Serialization for objects of type Rat, FatRat, Complex, BigInteger, Int3...
authored
192 Rat,
193 FatRat,
194 Complex,
195 BigInteger,
196 VarDeque,
197 VarHash,
dc6dd55 @sorear Implement serialization for Frame, finally kill off hashtable lexicals
authored
198 DispatchEnt,
199 RxFrame,
200 P6how,
1993029 @sorear Start implementing serialization for our objects
authored
201
202 // types of P6any-reified object
d91ecdf @sorear Serialization for objects of type Rat, FatRat, Complex, BigInteger, Int3...
authored
203 P6opaque, // eventually let's specialize this
1993029 @sorear Start implementing serialization for our objects
authored
204 Frame,
205 Cursor,
206
d91ecdf @sorear Serialization for objects of type Rat, FatRat, Complex, BigInteger, Int3...
authored
207 // miscellany
208 Variant, // allow 5, see FallbackFreeze
209
ab35238 @sorear Tidy up STable fields a bit, add TiedVariable serialize
authored
210 // variables
211 SimpleVariable = Variant + 5, // allow 4 for flags
1993029 @sorear Start implementing serialization for our objects
authored
212 SubstrLValue = SimpleVariable + 4,
ab35238 @sorear Tidy up STable fields a bit, add TiedVariable serialize
authored
213 TiedVariable,
1993029 @sorear Start implementing serialization for our objects
authored
214
215 // vivification hooks
ab35238 @sorear Tidy up STable fields a bit, add TiedVariable serialize
authored
216 SubViviHook,
1993029 @sorear Start implementing serialization for our objects
authored
217 ArrayViviHook,
218 NewArrayViviHook,
219 HashViviHook,
220 NewHashViviHook,
b1088dc @sorear implement serialization for SubInfo, LAD, LexInfo
authored
221
222 // Longest-token automaton descriptors
223 LADNone, // no-args
224 LADNull,
225 LADDot,
226 LADDispatcher,
227 LADImp,
228 LADStr, // string
229 LADStrNoCase,
230 LADMethod,
231 LADParam,
232 LADOpt, // LAD
233 LADPlus,
234 LADStar,
235 LADSequence, // LAD[]
236 LADAny,
237 LADCC, // CC
8184302 @sorear Start draft of serialization/deserialization code
authored
238 }
239
240 // An instance of this class is used to serialize serialization units
1993029 @sorear Start implementing serialization for our objects
authored
241 public class FreezeBuffer {
8184302 @sorear Start draft of serialization/deserialization code
authored
242 byte[] data;
243 int wpointer;
244
d832c02 @sorear Second draft of serialization code
authored
245 Dictionary<SerUnit,int> unit_to_offset;
246 int usedunits;
8184302 @sorear Start draft of serialization/deserialization code
authored
247
248 ObjectRegistry reg;
d832c02 @sorear Second draft of serialization code
authored
249 SerUnit unit;
8184302 @sorear Start draft of serialization/deserialization code
authored
250
d832c02 @sorear Second draft of serialization code
authored
251 internal FreezeBuffer(ObjectRegistry reg, SerUnit unit) {
8ed36ea @sorear Wire up serializer to run after pre-compiler
authored
252 if (reg == null || unit == null)
253 throw new ArgumentNullException();
8184302 @sorear Start draft of serialization/deserialization code
authored
254 this.reg = reg;
d832c02 @sorear Second draft of serialization code
authored
255 this.unit = unit;
256 unit_to_offset = new Dictionary<SerUnit,int>();
8184302 @sorear Start draft of serialization/deserialization code
authored
257 data = new byte[256];
258 }
259
b25256f @sorear Add Serialize.cs to build, fix build errors
authored
260 internal byte[] GetData() {
261 byte[] ret = new byte[wpointer];
262 Array.Copy(data, ret, ret.Length);
263 return ret;
264 }
265
8184302 @sorear Start draft of serialization/deserialization code
authored
266 void Ensure(int ct) {
267 while (ct + wpointer > data.Length)
268 Array.Resize(ref data, data.Length * 2);
269 }
270
271 public void Byte(byte x) {
272 Ensure(1);
273 data[wpointer++] = x;
274 }
275
276 public void Short(short x) {
277 Ensure(2);
278 data[wpointer++] = (byte)(x >> 8);
279 data[wpointer++] = (byte)(x );
280 }
281
282 public void Int(int x) {
283 Ensure(4);
284 data[wpointer++] = (byte)(x >> 24);
285 data[wpointer++] = (byte)(x >> 16);
286 data[wpointer++] = (byte)(x >> 8);
287 data[wpointer++] = (byte)(x );
288 }
289
290 public void Long(long x) {
291 Ensure(8);
292 data[wpointer++] = (byte)(x >> 56);
293 data[wpointer++] = (byte)(x >> 48);
294 data[wpointer++] = (byte)(x >> 40);
295 data[wpointer++] = (byte)(x >> 32);
296 data[wpointer++] = (byte)(x >> 24);
297 data[wpointer++] = (byte)(x >> 16);
298 data[wpointer++] = (byte)(x >> 8);
299 data[wpointer++] = (byte)(x );
300 }
301
302 public void String(string s) {
303 if (s == null) {
304 Int(-1);
305 } else {
306 Int(s.Length);
307 foreach (char ch in s)
308 Short((short)ch);
309 }
310 }
311
3dc4789 @sorear Implement freezing for types
authored
312 public void Strings(string[] s) {
313 if (s == null) Int(-1);
314 else {
315 Int(s.Length);
316 foreach (string ch in s) String(ch);
317 }
318 }
319
b1088dc @sorear implement serialization for SubInfo, LAD, LexInfo
authored
320 public void Ints(int[] s) {
321 if (s == null) {
322 Int(-1);
323 } else {
324 Int(s.Length);
325 foreach (int ch in s)
326 Int(ch);
327 }
328 }
329
dc6dd55 @sorear Implement serialization for Frame, finally kill off hashtable lexicals
authored
330 public void Refs<T> (T[] x) {
d91ecdf @sorear Serialization for objects of type Rat, FatRat, Complex, BigInteger, Int3...
authored
331 if (x == null) {
332 Int(-1);
333 } else {
334 Int(x.Length);
335 foreach (T y in x)
336 ObjRef(y);
337 }
338 }
339
dc6dd55 @sorear Implement serialization for Frame, finally kill off hashtable lexicals
authored
340 public void Refs<T> (IList<T> x) {
3dc4789 @sorear Implement freezing for types
authored
341 if (x == null) {
342 Int(-1);
343 } else {
344 Int(x.Count);
345 foreach (T y in x)
346 ObjRef(y);
347 }
348 }
349
d832c02 @sorear Second draft of serialization code
authored
350 // This is the main routine you should call from your Freeze
351 // callbacks to freeze an object
d91ecdf @sorear Serialization for objects of type Rat, FatRat, Complex, BigInteger, Int3...
authored
352 public void ObjRef(object o) {
d832c02 @sorear Second draft of serialization code
authored
353 int id;
354 SerUnit altunit;
355 if (o == null) { // null pointers are special
356 Byte((byte)SerializationCode.Null);
357 return;
358 }
359
360 if (reg.CheckWriteObject(unit, o, out altunit, out id)) {
8184302 @sorear Start draft of serialization/deserialization code
authored
361 if (altunit == unit) {
362 Byte((byte)SerializationCode.SelfRef);
363 } else {
d832c02 @sorear Second draft of serialization code
authored
364 int altcode;
365 if (!unit_to_offset.TryGetValue(altunit, out altcode)) {
8184302 @sorear Start draft of serialization/deserialization code
authored
366 Byte((byte)SerializationCode.NewUnitRef);
b25256f @sorear Add Serialize.cs to build, fix build errors
authored
367 String(altunit.name);
d832c02 @sorear Second draft of serialization code
authored
368 // save the hash too so stale refs can be caught
369 foreach (byte b in altunit.hash) Byte(b);
370
8184302 @sorear Start draft of serialization/deserialization code
authored
371 unit_to_offset[altunit] = usedunits++;
372 } else {
373 Byte((byte)SerializationCode.ForeignRef);
d832c02 @sorear Second draft of serialization code
authored
374 Int(altcode);
8184302 @sorear Start draft of serialization/deserialization code
authored
375 }
376 }
377 Int((int)id);
378 } else {
d832c02 @sorear Second draft of serialization code
authored
379 // must take responsibility for saving the tag
d91ecdf @sorear Serialization for objects of type Rat, FatRat, Complex, BigInteger, Int3...
authored
380 IFreeze f = o as IFreeze;
381 if (f != null) {
382 f.Freeze(this);
383 } else {
384 FallbackFreeze(o);
385 }
386 }
387 }
388
389 // Call this to freeze a variant-typed value. (Avoid)
390 static Type[] anyTypes = new Type[] {
391 typeof(string), typeof(P6any[]), typeof(Variable[]),
392 typeof(int), typeof(double),
393 };
394
395 void FallbackFreeze(object o) {
396 int ix = 0;
397 Type t = o.GetType();
398 while (ix != 11 && anyTypes[ix] != t) ix++;
399 Byte((byte)(((int)SerializationCode.Variant) + ix));
400
401 switch(ix) {
402 case 0:
403 String((string)o);
404 break;
405 case 1:
406 Refs((P6any[])o);
407 break;
408 case 2:
409 Refs((Variable[])o);
410 break;
411 case 3:
412 Int((int)o);
413 break;
414 case 4:
415 Long(BitConverter.DoubleToInt64Bits((double)o));
416 break;
417 default:
418 throw new NotImplementedException(t.FullName);
8184302 @sorear Start draft of serialization/deserialization code
authored
419 }
420 }
421 }
422
423 // Note that this interface only handles freezing - thaw is done using
424 // a switch statement.
1993029 @sorear Start implementing serialization for our objects
authored
425 public interface IFreeze {
8184302 @sorear Start draft of serialization/deserialization code
authored
426 void Freeze(FreezeBuffer fb);
427 }
428
429 class ThawBuffer {
430 byte[] data;
431 int rpointer;
432 ObjectRegistry reg;
433
d832c02 @sorear Second draft of serialization code
authored
434 SerUnit[] unit_map = new SerUnit[8];
8184302 @sorear Start draft of serialization/deserialization code
authored
435 int refed_units;
d832c02 @sorear Second draft of serialization code
authored
436 SerUnit unit;
8184302 @sorear Start draft of serialization/deserialization code
authored
437
b25256f @sorear Add Serialize.cs to build, fix build errors
authored
438 internal ThawBuffer(ObjectRegistry reg, SerUnit unit, byte[] data) {
8184302 @sorear Start draft of serialization/deserialization code
authored
439 this.data = data;
440 this.reg = reg;
d832c02 @sorear Second draft of serialization code
authored
441 this.unit = unit;
8184302 @sorear Start draft of serialization/deserialization code
authored
442 }
443
b25256f @sorear Add Serialize.cs to build, fix build errors
authored
444 public byte Byte() { return data[rpointer++]; }
445
446 public short Short() {
447 return (short)((((int)Byte()) << 8) | Byte());
448 }
449
450 public int Int() {
451 return (((int)Byte()) << 24) | (((int)Byte()) << 16) |
452 (((int)Byte()) << 8) | ((int)Byte());
453 }
454
455 public long Long() {
456 // try to do as much as possible in 32-bit precision,
457 // but suppress sign extension
458 return (((long)Int()) << 32) | (long)(uint)Int();
459 }
460
461 public string String() {
462 int l = Int();
463
464 if (l < 0) return null;
465 char[] cb = new char[l];
466
467 for (int i = 0; i < l; i++)
468 cb[i] = (char)Short();
469
470 return new string(cb);
471 }
472
473 public byte[] Bytes(int k) {
474 byte[] buf = new byte[k];
475
476 for (int i = 0; i < k; i++)
477 buf[i] = Byte();
478
479 return buf;
480 }
481
482 public object ObjRef() {
8184302 @sorear Start draft of serialization/deserialization code
authored
483 var tag = (SerializationCode)Byte();
484 int i, j;
485 switch(tag) {
d832c02 @sorear Second draft of serialization code
authored
486 case SerializationCode.Null:
487 return null;
8184302 @sorear Start draft of serialization/deserialization code
authored
488 case SerializationCode.SelfRef:
489 i = Int();
d832c02 @sorear Second draft of serialization code
authored
490 return unit.bynum[i];
8184302 @sorear Start draft of serialization/deserialization code
authored
491 case SerializationCode.ForeignRef:
492 i = Int();
493 j = Int();
d832c02 @sorear Second draft of serialization code
authored
494 return unit_map[i].bynum[j];
8184302 @sorear Start draft of serialization/deserialization code
authored
495 case SerializationCode.NewUnitRef:
d832c02 @sorear Second draft of serialization code
authored
496 return LoadNewUnit();
8184302 @sorear Start draft of serialization/deserialization code
authored
497 default:
498 throw new ThawException("unexpected object tag" + (byte)tag);
499 }
500 }
d832c02 @sorear Second draft of serialization code
authored
501
502 object LoadNewUnit() {
503 string name = String();
504 if (refed_units == unit_map.Length)
505 Array.Resize(ref unit_map, refed_units * 2);
506
507 SerUnit su = reg.LoadUnit(name);
508 unit_map[refed_units++] = su;
509
510 byte[] hash = Bytes(su.hash.Length);
511
512 for (int i = 0; i < hash.Length; i++)
513 if (hash[i] != su.hash[i])
514 goto badhash;
515
516 int ix = Int();
517 return su.bynum[ix];
518
519 badhash:
520 StringBuilder sb = new StringBuilder();
521 sb.AppendFormat("Hash mismatch for unit {0} referenced from {1}",
522 su.name, unit.name);
523
524 sb.Append(", wanted ");
525 foreach (byte b in hash)
526 sb.AppendFormat("{0:X2}", b);
527
528 sb.Append(", got ");
529 foreach (byte b in su.hash)
530 sb.AppendFormat("{0:X2}", b);
531
532 throw new ThawException(sb.ToString());
533 }
534 }
535
536 // Thrown to indicate data format problems in the serialized stream
537 // Not necessarily bugs; could also indicate stale files, including
538 // cases where the data format is changed and cases where a depended
539 // file was recreated
540 class ThawException : Exception {
541 public ThawException(string s) : base(s) { }
542 public ThawException() : base() { }
8184302 @sorear Start draft of serialization/deserialization code
authored
543 }
544 }
Something went wrong with that request. Please try again.