Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Newer
Older
100644 516 lines (432 sloc) 15.912 kB
8184302 @sorear Start draft of serialization/deserialization code
authored
1 using System;
d832c02 @sorear Second draft of serialization code
authored
2 using System.IO;
3 using System.Security.Cryptography;
b25256f @sorear Add Serialize.cs to build, fix build errors
authored
4 using System.Collections.Generic;
d832c02 @sorear Second draft of serialization code
authored
5 using System.Text;
8184302 @sorear Start draft of serialization/deserialization code
authored
6
cdde7e0 @sorear Notes on four kinds of module scope
authored
7 // Here in Niecza we have four different kinds of unit scopes:
8 //
9 // * COMPILING::UNIT, aka RuntimeUnit: one of these exists for every
10 // call into the compiler, whether eval or module. The REPL will
11 // be considered as if it were eval.
12 //
13 // * Serialization scopes, which are created when compiling modules
14 // or when pre-compiling a main program. During execution there is
15 // no current serialization scope; evals inherit the serialization
16 // scope or lack thereof that was in effect.
17 //
18 // * Assembly scopes, which are an artifact of the CLR and almost align
19 // with precompilation scopes, except that they have to exist always
20 // because methods cannot be created free-floating in CLR 2.x.
21 //
22 // An assembly scope is created for all serialization scopes, and
23 // non-saved anonymous assemblies are created for eval-and-run of
24 // a file and non-BEGIN-time evals.
25 //
26 // * GLOBAL scope is very much like serialization scope except that there
27 // is a "true globals" scope that is used when not serializing.
28
8184302 @sorear Start draft of serialization/deserialization code
authored
29 // This implements "bounded serialization" for Niecza. Unfortunately
30 // the CLR's builtin serialization can't efficiently be made bounded,
31 // and anyway it would be nice if the serialization format could be
32 // transported across backends.
33
34 // TODO: implement a more Storable-like interface.
d832c02 @sorear Second draft of serialization code
authored
35
36 // Note, the serialization subsystem is *NOT* thread safe !
8184302 @sorear Start draft of serialization/deserialization code
authored
37 namespace Niecza.Serialization {
d832c02 @sorear Second draft of serialization code
authored
38 // Information kept on a serialization unit after loading or storing,
39 // but not before storing.
40 class SerUnit {
41 internal string name; // eg "File.Copy"
42 internal byte[] hash; // hash of entire file, filled at write time
43 internal object[] bynum; // objects in unit
44 internal object root; // the RuntimeUnit object
45 internal int nobj;
46 }
8184302 @sorear Start draft of serialization/deserialization code
authored
47
48 // The central feature of *bounded* serialization is that object
49 // registries are kept distinct from the (de)serializer, and can
50 // be shared between serialization runs.
51 class ObjectRegistry {
d832c02 @sorear Second draft of serialization code
authored
52 // TODO: investigate a more specialized representation,
53 // ideally not having to hash as many objects
54 struct ObjRef {
b25256f @sorear Add Serialize.cs to build, fix build errors
authored
55 public SerUnit unit;
56 public int id;
8184302 @sorear Start draft of serialization/deserialization code
authored
57 }
d832c02 @sorear Second draft of serialization code
authored
58 Dictionary<object,ObjRef> byref = new Dictionary<object,ObjRef>();
8184302 @sorear Start draft of serialization/deserialization code
authored
59
d832c02 @sorear Second draft of serialization code
authored
60 Dictionary<string,SerUnit> units =
61 new Dictionary<string,SerUnit>();
62
63 static readonly HashAlgorithm hash = SHA256.Create();
64 static readonly string signature = "Niecza-Serialized-Module";
65 static readonly int version = 1;
66
67 // Routines for use by serialization code
68 public bool CheckWriteObject(SerUnit into, object o,
69 out SerUnit lui, out int id) {
70 ObjRef or;
71 if (byref.TryGetValue(o, out or)) {
72 lui = or.unit;
73 id = or.id;
8184302 @sorear Start draft of serialization/deserialization code
authored
74 return true;
d832c02 @sorear Second draft of serialization code
authored
75 }
8184302 @sorear Start draft of serialization/deserialization code
authored
76
d832c02 @sorear Second draft of serialization code
authored
77 if (into.nobj == into.bynum.Length)
78 Array.Resize(ref into.bynum, into.nobj * 2);
8184302 @sorear Start draft of serialization/deserialization code
authored
79
b25256f @sorear Add Serialize.cs to build, fix build errors
authored
80 or.unit = lui = into;
d832c02 @sorear Second draft of serialization code
authored
81 id = or.id = into.nobj++;
82 into.bynum[id] = o;
83
84 byref[o] = or;
8184302 @sorear Start draft of serialization/deserialization code
authored
85
86 return false;
87 }
d832c02 @sorear Second draft of serialization code
authored
88
89 // Routines for use by compilation manager
90
91 // Loads a single unit from the compiled-data directory.
92 // Will throw a ThawException if a stale reference is encountered
93 // or other data format error.
94 public SerUnit LoadUnit(string name) {
95 SerUnit su;
96
97 // is the unit already loaded?
98 if (units.TryGetValue(name, out su))
99 return su;
100
101 string file = Path.Combine(AppDomain.CurrentDomain.BaseDirectory,
102 name + ".ser");
103 byte[] bytes = File.ReadAllBytes(file);
104
105 su = new SerUnit();
106 su.name = name;
107 su.hash = hash.ComputeHash(bytes);
108
b25256f @sorear Add Serialize.cs to build, fix build errors
authored
109 ThawBuffer tb = new ThawBuffer(this, su, bytes);
d832c02 @sorear Second draft of serialization code
authored
110
111 bool success = false;
112 try {
113 string rsig = tb.String();
114 if (rsig != signature)
115 throw new ThawException("signature mismatch loading " + file);
116 int rver = tb.Int();
117 if (rver != version)
118 throw new ThawException("version mismatch loading " + file);
119
b25256f @sorear Add Serialize.cs to build, fix build errors
authored
120 su.root = tb.ObjRef();
d832c02 @sorear Second draft of serialization code
authored
121 success = true;
122 } finally {
123 // don't leave half-read units in the map
124 if (!success)
125 UnloadUnit(name);
126 }
127
128 return su;
129 }
130
131 // removes a stale unit so a new version can be saved over it.
132 public void UnloadUnit(string name) {
133 SerUnit su = units[name];
134 units.Remove(name);
135
136 for (int i = 0; i < su.nobj; i++)
137 byref.Remove(su.bynum[i]);
138 }
139
140 public SerUnit SaveUnit(string name, IFreeze root) {
141 SerUnit su = new SerUnit();
142 su.name = name;
143 su.root = root;
144
145 if (units.ContainsKey(name))
b25256f @sorear Add Serialize.cs to build, fix build errors
authored
146 throw new InvalidOperationException("unit " +name+ " exists");
d832c02 @sorear Second draft of serialization code
authored
147
148 bool success = false;
149 string file = Path.Combine(AppDomain.CurrentDomain.BaseDirectory,
150 name + ".ser");
151
152 FreezeBuffer fb = new FreezeBuffer(this, su);
153
154 try {
155 fb.String(signature);
156 fb.Int(version);
157 fb.ObjRef(root);
158
159 byte[] data = fb.GetData();
160 su.hash = hash.ComputeHash(data);
161 File.WriteAllBytes(file, data);
162 success = true;
163 } finally {
164 if (!success)
165 UnloadUnit(name);
166 }
167
168 return su;
169 }
8184302 @sorear Start draft of serialization/deserialization code
authored
170 }
171
172 // One of these codes is written at the beginning of every object ref
1993029 @sorear Start implementing serialization for our objects
authored
173 enum SerializationCode : byte {
d832c02 @sorear Second draft of serialization code
authored
174 // special
175 Null,
176
8184302 @sorear Start draft of serialization/deserialization code
authored
177 // existing objects
178 ForeignRef,
179 SelfRef,
180 NewUnitRef,
1993029 @sorear Start implementing serialization for our objects
authored
181
182 // types of new object
183 RuntimeUnit,
184 SubInfo,
185 STable,
186 StashEnt,
d91ecdf @sorear Serialization for objects of type Rat, FatRat, Complex, BigInteger, I…
authored
187 Rat,
188 FatRat,
189 Complex,
190 BigInteger,
191 VarDeque,
192 VarHash,
1993029 @sorear Start implementing serialization for our objects
authored
193
194 // types of P6any-reified object
d91ecdf @sorear Serialization for objects of type Rat, FatRat, Complex, BigInteger, I…
authored
195 P6opaque, // eventually let's specialize this
1993029 @sorear Start implementing serialization for our objects
authored
196 Frame,
197 Cursor,
198
d91ecdf @sorear Serialization for objects of type Rat, FatRat, Complex, BigInteger, I…
authored
199 // miscellany
200 Variant, // allow 5, see FallbackFreeze
201
1993029 @sorear Start implementing serialization for our objects
authored
202 // variables - allow 4 codes each for flag compaction
d91ecdf @sorear Serialization for objects of type Rat, FatRat, Complex, BigInteger, I…
authored
203 SimpleVariable = Variant + 5,
1993029 @sorear Start implementing serialization for our objects
authored
204 SubstrLValue = SimpleVariable + 4,
205
206 // vivification hooks
207 SubViviHook = SubstrLValue + 4,
208 ArrayViviHook,
209 NewArrayViviHook,
210 HashViviHook,
211 NewHashViviHook,
b1088dc @sorear implement serialization for SubInfo, LAD, LexInfo
authored
212
213 // Longest-token automaton descriptors
214 LADNone, // no-args
215 LADNull,
216 LADDot,
217 LADDispatcher,
218 LADImp,
219 LADStr, // string
220 LADStrNoCase,
221 LADMethod,
222 LADParam,
223 LADOpt, // LAD
224 LADPlus,
225 LADStar,
226 LADSequence, // LAD[]
227 LADAny,
228 LADCC, // CC
8184302 @sorear Start draft of serialization/deserialization code
authored
229 }
230
231 // An instance of this class is used to serialize serialization units
1993029 @sorear Start implementing serialization for our objects
authored
232 public class FreezeBuffer {
8184302 @sorear Start draft of serialization/deserialization code
authored
233 byte[] data;
234 int wpointer;
235
d832c02 @sorear Second draft of serialization code
authored
236 Dictionary<SerUnit,int> unit_to_offset;
237 int usedunits;
8184302 @sorear Start draft of serialization/deserialization code
authored
238
239 ObjectRegistry reg;
d832c02 @sorear Second draft of serialization code
authored
240 SerUnit unit;
8184302 @sorear Start draft of serialization/deserialization code
authored
241
d832c02 @sorear Second draft of serialization code
authored
242 internal FreezeBuffer(ObjectRegistry reg, SerUnit unit) {
8184302 @sorear Start draft of serialization/deserialization code
authored
243 this.reg = reg;
d832c02 @sorear Second draft of serialization code
authored
244 this.unit = unit;
245 unit_to_offset = new Dictionary<SerUnit,int>();
8184302 @sorear Start draft of serialization/deserialization code
authored
246 data = new byte[256];
247 }
248
b25256f @sorear Add Serialize.cs to build, fix build errors
authored
249 internal byte[] GetData() {
250 byte[] ret = new byte[wpointer];
251 Array.Copy(data, ret, ret.Length);
252 return ret;
253 }
254
8184302 @sorear Start draft of serialization/deserialization code
authored
255 void Ensure(int ct) {
256 while (ct + wpointer > data.Length)
257 Array.Resize(ref data, data.Length * 2);
258 }
259
260 public void Byte(byte x) {
261 Ensure(1);
262 data[wpointer++] = x;
263 }
264
265 public void Short(short x) {
266 Ensure(2);
267 data[wpointer++] = (byte)(x >> 8);
268 data[wpointer++] = (byte)(x );
269 }
270
271 public void Int(int x) {
272 Ensure(4);
273 data[wpointer++] = (byte)(x >> 24);
274 data[wpointer++] = (byte)(x >> 16);
275 data[wpointer++] = (byte)(x >> 8);
276 data[wpointer++] = (byte)(x );
277 }
278
279 public void Long(long x) {
280 Ensure(8);
281 data[wpointer++] = (byte)(x >> 56);
282 data[wpointer++] = (byte)(x >> 48);
283 data[wpointer++] = (byte)(x >> 40);
284 data[wpointer++] = (byte)(x >> 32);
285 data[wpointer++] = (byte)(x >> 24);
286 data[wpointer++] = (byte)(x >> 16);
287 data[wpointer++] = (byte)(x >> 8);
288 data[wpointer++] = (byte)(x );
289 }
290
291 public void String(string s) {
292 if (s == null) {
293 Int(-1);
294 } else {
295 Int(s.Length);
296 foreach (char ch in s)
297 Short((short)ch);
298 }
299 }
300
b1088dc @sorear implement serialization for SubInfo, LAD, LexInfo
authored
301 public void Ints(int[] s) {
302 if (s == null) {
303 Int(-1);
304 } else {
305 Int(s.Length);
306 foreach (int ch in s)
307 Int(ch);
308 }
309 }
310
d91ecdf @sorear Serialization for objects of type Rat, FatRat, Complex, BigInteger, I…
authored
311 public void Refs<T> (T[] x) where T: IFreeze {
312 if (x == null) {
313 Int(-1);
314 } else {
315 Int(x.Length);
316 foreach (T y in x)
317 ObjRef(y);
318 }
319 }
320
d832c02 @sorear Second draft of serialization code
authored
321 // This is the main routine you should call from your Freeze
322 // callbacks to freeze an object
d91ecdf @sorear Serialization for objects of type Rat, FatRat, Complex, BigInteger, I…
authored
323 public void ObjRef(object o) {
d832c02 @sorear Second draft of serialization code
authored
324 int id;
325 SerUnit altunit;
326 if (o == null) { // null pointers are special
327 Byte((byte)SerializationCode.Null);
328 return;
329 }
330
331 if (reg.CheckWriteObject(unit, o, out altunit, out id)) {
8184302 @sorear Start draft of serialization/deserialization code
authored
332 if (altunit == unit) {
333 Byte((byte)SerializationCode.SelfRef);
334 } else {
d832c02 @sorear Second draft of serialization code
authored
335 int altcode;
336 if (!unit_to_offset.TryGetValue(altunit, out altcode)) {
8184302 @sorear Start draft of serialization/deserialization code
authored
337 Byte((byte)SerializationCode.NewUnitRef);
b25256f @sorear Add Serialize.cs to build, fix build errors
authored
338 String(altunit.name);
d832c02 @sorear Second draft of serialization code
authored
339 // save the hash too so stale refs can be caught
340 foreach (byte b in altunit.hash) Byte(b);
341
8184302 @sorear Start draft of serialization/deserialization code
authored
342 unit_to_offset[altunit] = usedunits++;
343 } else {
344 Byte((byte)SerializationCode.ForeignRef);
d832c02 @sorear Second draft of serialization code
authored
345 Int(altcode);
8184302 @sorear Start draft of serialization/deserialization code
authored
346 }
347 }
348 Int((int)id);
349 } else {
d832c02 @sorear Second draft of serialization code
authored
350 // must take responsibility for saving the tag
d91ecdf @sorear Serialization for objects of type Rat, FatRat, Complex, BigInteger, I…
authored
351 IFreeze f = o as IFreeze;
352 if (f != null) {
353 f.Freeze(this);
354 } else {
355 FallbackFreeze(o);
356 }
357 }
358 }
359
360 // Call this to freeze a variant-typed value. (Avoid)
361 static Type[] anyTypes = new Type[] {
362 typeof(string), typeof(P6any[]), typeof(Variable[]),
363 typeof(int), typeof(double),
364 };
365
366 void FallbackFreeze(object o) {
367 int ix = 0;
368 Type t = o.GetType();
369 while (ix != 11 && anyTypes[ix] != t) ix++;
370 Byte((byte)(((int)SerializationCode.Variant) + ix));
371
372 switch(ix) {
373 case 0:
374 String((string)o);
375 break;
376 case 1:
377 Refs((P6any[])o);
378 break;
379 case 2:
380 Refs((Variable[])o);
381 break;
382 case 3:
383 Int((int)o);
384 break;
385 case 4:
386 Long(BitConverter.DoubleToInt64Bits((double)o));
387 break;
388 default:
389 throw new NotImplementedException(t.FullName);
8184302 @sorear Start draft of serialization/deserialization code
authored
390 }
391 }
392 }
393
394 // Note that this interface only handles freezing - thaw is done using
395 // a switch statement.
1993029 @sorear Start implementing serialization for our objects
authored
396 public interface IFreeze {
8184302 @sorear Start draft of serialization/deserialization code
authored
397 void Freeze(FreezeBuffer fb);
398 }
399
400 class ThawBuffer {
401 byte[] data;
402 int rpointer;
403 ObjectRegistry reg;
404
d832c02 @sorear Second draft of serialization code
authored
405 SerUnit[] unit_map = new SerUnit[8];
8184302 @sorear Start draft of serialization/deserialization code
authored
406 int refed_units;
d832c02 @sorear Second draft of serialization code
authored
407 SerUnit unit;
8184302 @sorear Start draft of serialization/deserialization code
authored
408
b25256f @sorear Add Serialize.cs to build, fix build errors
authored
409 internal ThawBuffer(ObjectRegistry reg, SerUnit unit, byte[] data) {
8184302 @sorear Start draft of serialization/deserialization code
authored
410 this.data = data;
411 this.reg = reg;
d832c02 @sorear Second draft of serialization code
authored
412 this.unit = unit;
8184302 @sorear Start draft of serialization/deserialization code
authored
413 }
414
b25256f @sorear Add Serialize.cs to build, fix build errors
authored
415 public byte Byte() { return data[rpointer++]; }
416
417 public short Short() {
418 return (short)((((int)Byte()) << 8) | Byte());
419 }
420
421 public int Int() {
422 return (((int)Byte()) << 24) | (((int)Byte()) << 16) |
423 (((int)Byte()) << 8) | ((int)Byte());
424 }
425
426 public long Long() {
427 // try to do as much as possible in 32-bit precision,
428 // but suppress sign extension
429 return (((long)Int()) << 32) | (long)(uint)Int();
430 }
431
432 public string String() {
433 int l = Int();
434
435 if (l < 0) return null;
436 char[] cb = new char[l];
437
438 for (int i = 0; i < l; i++)
439 cb[i] = (char)Short();
440
441 return new string(cb);
442 }
443
444 public byte[] Bytes(int k) {
445 byte[] buf = new byte[k];
446
447 for (int i = 0; i < k; i++)
448 buf[i] = Byte();
449
450 return buf;
451 }
452
453 public object ObjRef() {
8184302 @sorear Start draft of serialization/deserialization code
authored
454 var tag = (SerializationCode)Byte();
455 int i, j;
456 switch(tag) {
d832c02 @sorear Second draft of serialization code
authored
457 case SerializationCode.Null:
458 return null;
8184302 @sorear Start draft of serialization/deserialization code
authored
459 case SerializationCode.SelfRef:
460 i = Int();
d832c02 @sorear Second draft of serialization code
authored
461 return unit.bynum[i];
8184302 @sorear Start draft of serialization/deserialization code
authored
462 case SerializationCode.ForeignRef:
463 i = Int();
464 j = Int();
d832c02 @sorear Second draft of serialization code
authored
465 return unit_map[i].bynum[j];
8184302 @sorear Start draft of serialization/deserialization code
authored
466 case SerializationCode.NewUnitRef:
d832c02 @sorear Second draft of serialization code
authored
467 return LoadNewUnit();
8184302 @sorear Start draft of serialization/deserialization code
authored
468 default:
469 throw new ThawException("unexpected object tag" + (byte)tag);
470 }
471 }
d832c02 @sorear Second draft of serialization code
authored
472
473 object LoadNewUnit() {
474 string name = String();
475 if (refed_units == unit_map.Length)
476 Array.Resize(ref unit_map, refed_units * 2);
477
478 SerUnit su = reg.LoadUnit(name);
479 unit_map[refed_units++] = su;
480
481 byte[] hash = Bytes(su.hash.Length);
482
483 for (int i = 0; i < hash.Length; i++)
484 if (hash[i] != su.hash[i])
485 goto badhash;
486
487 int ix = Int();
488 return su.bynum[ix];
489
490 badhash:
491 StringBuilder sb = new StringBuilder();
492 sb.AppendFormat("Hash mismatch for unit {0} referenced from {1}",
493 su.name, unit.name);
494
495 sb.Append(", wanted ");
496 foreach (byte b in hash)
497 sb.AppendFormat("{0:X2}", b);
498
499 sb.Append(", got ");
500 foreach (byte b in su.hash)
501 sb.AppendFormat("{0:X2}", b);
502
503 throw new ThawException(sb.ToString());
504 }
505 }
506
507 // Thrown to indicate data format problems in the serialized stream
508 // Not necessarily bugs; could also indicate stale files, including
509 // cases where the data format is changed and cases where a depended
510 // file was recreated
511 class ThawException : Exception {
512 public ThawException(string s) : base(s) { }
513 public ThawException() : base() { }
8184302 @sorear Start draft of serialization/deserialization code
authored
514 }
515 }
Something went wrong with that request. Please try again.