Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Newer
Older
100644 858 lines (745 sloc) 28.188 kB
8184302 @sorear Start draft of serialization/deserialization code
authored
1 using System;
d832c02 @sorear Second draft of serialization code
authored
2 using System.IO;
3 using System.Security.Cryptography;
b25256f @sorear Add Serialize.cs to build, fix build errors
authored
4 using System.Collections.Generic;
d832c02 @sorear Second draft of serialization code
authored
5 using System.Text;
9083142 @sorear Switch to single-AppDomain execution
authored
6 using Niecza.CLRBackend;
8184302 @sorear Start draft of serialization/deserialization code
authored
7
cdde7e0 @sorear Notes on four kinds of module scope
authored
8 // Here in Niecza we have four different kinds of unit scopes:
9 //
10 // * COMPILING::UNIT, aka RuntimeUnit: one of these exists for every
11 // call into the compiler, whether eval or module. The REPL will
12 // be considered as if it were eval.
13 //
14 // * Serialization scopes, which are created when compiling modules
15 // or when pre-compiling a main program. During execution there is
16 // no current serialization scope; evals inherit the serialization
17 // scope or lack thereof that was in effect.
18 //
19 // * Assembly scopes, which are an artifact of the CLR and almost align
20 // with precompilation scopes, except that they have to exist always
21 // because methods cannot be created free-floating in CLR 2.x.
22 //
23 // An assembly scope is created for all serialization scopes, and
24 // non-saved anonymous assemblies are created for eval-and-run of
25 // a file and non-BEGIN-time evals.
26 //
27 // * GLOBAL scope is very much like serialization scope except that there
28 // is a "true globals" scope that is used when not serializing.
29
8184302 @sorear Start draft of serialization/deserialization code
authored
30 // This implements "bounded serialization" for Niecza. Unfortunately
31 // the CLR's builtin serialization can't efficiently be made bounded,
32 // and anyway it would be nice if the serialization format could be
33 // transported across backends.
34
35 // TODO: implement a more Storable-like interface.
d832c02 @sorear Second draft of serialization code
authored
36
37 // Note, the serialization subsystem is *NOT* thread safe !
8184302 @sorear Start draft of serialization/deserialization code
authored
38 namespace Niecza.Serialization {
d832c02 @sorear Second draft of serialization code
authored
39 // Information kept on a serialization unit after loading or storing,
40 // but not before storing.
41 class SerUnit {
42 internal string name; // eg "File.Copy"
43 internal byte[] hash; // hash of entire file, filled at write time
8ed36ea @sorear Wire up serializer to run after pre-compiler
authored
44 internal object[] bynum = new object[8]; // objects in unit
d832c02 @sorear Second draft of serialization code
authored
45 internal object root; // the RuntimeUnit object
8ed36ea @sorear Wire up serializer to run after pre-compiler
authored
46 internal int nobj; // = 0
d832c02 @sorear Second draft of serialization code
authored
47 }
8184302 @sorear Start draft of serialization/deserialization code
authored
48
49 // The central feature of *bounded* serialization is that object
50 // registries are kept distinct from the (de)serializer, and can
51 // be shared between serialization runs.
52 class ObjectRegistry {
d832c02 @sorear Second draft of serialization code
authored
53 // TODO: investigate a more specialized representation,
54 // ideally not having to hash as many objects
55 struct ObjRef {
b25256f @sorear Add Serialize.cs to build, fix build errors
authored
56 public SerUnit unit;
57 public int id;
8184302 @sorear Start draft of serialization/deserialization code
authored
58 }
d832c02 @sorear Second draft of serialization code
authored
59 Dictionary<object,ObjRef> byref = new Dictionary<object,ObjRef>();
8184302 @sorear Start draft of serialization/deserialization code
authored
60
d832c02 @sorear Second draft of serialization code
authored
61 Dictionary<string,SerUnit> units =
62 new Dictionary<string,SerUnit>();
63
cdb4548 @sorear Checkpoint of global-variable inventory
authored
64 internal static HashAlgorithm NewHash() { return new SHA256Managed(); }
65
d832c02 @sorear Second draft of serialization code
authored
66 static readonly string signature = "Niecza-Serialized-Module";
037f4b7 @sorear Make * a "blackhole" container that ignores writes
authored
67 static readonly int version = 21;
d832c02 @sorear Second draft of serialization code
authored
68
69 // Routines for use by serialization code
70 public bool CheckWriteObject(SerUnit into, object o,
71 out SerUnit lui, out int id) {
72 ObjRef or;
73 if (byref.TryGetValue(o, out or)) {
74 lui = or.unit;
75 id = or.id;
8184302 @sorear Start draft of serialization/deserialization code
authored
76 return true;
d832c02 @sorear Second draft of serialization code
authored
77 }
8184302 @sorear Start draft of serialization/deserialization code
authored
78
d832c02 @sorear Second draft of serialization code
authored
79 if (into.nobj == into.bynum.Length)
80 Array.Resize(ref into.bynum, into.nobj * 2);
8184302 @sorear Start draft of serialization/deserialization code
authored
81
b25256f @sorear Add Serialize.cs to build, fix build errors
authored
82 or.unit = lui = into;
d832c02 @sorear Second draft of serialization code
authored
83 id = or.id = into.nobj++;
84 into.bynum[id] = o;
85
86 byref[o] = or;
8184302 @sorear Start draft of serialization/deserialization code
authored
87
88 return false;
89 }
d832c02 @sorear Second draft of serialization code
authored
90
54921c8 @sorear Prevent choking when trying to serialize a multi-dispatch cache
authored
91 // Undo previous method...
92 public void DeleteLastObject(SerUnit into) {
93 byref.Remove(into.bynum[--into.nobj]);
94 into.bynum[into.nobj] = null;
95 }
96
1c5bb04 @sorear Small refactor to AssemblyBuilder setup, start thaw coding
authored
97 public void RegisterThawed(SerUnit into, object o) {
98 ObjRef or;
99 if (into.nobj == into.bynum.Length)
100 Array.Resize(ref into.bynum, into.nobj * 2);
101
102 or.unit = into;
103 or.id = into.nobj++;
104 into.bynum[or.id] = o;
105
106 byref[o] = or;
107 }
108
d832c02 @sorear Second draft of serialization code
authored
109 // Routines for use by compilation manager
110
111 // Loads a single unit from the compiled-data directory.
112 // Will throw a ThawException if a stale reference is encountered
113 // or other data format error.
114 public SerUnit LoadUnit(string name) {
115 SerUnit su;
116
117 // is the unit already loaded?
118 if (units.TryGetValue(name, out su))
119 return su;
120
9083142 @sorear Switch to single-AppDomain execution
authored
121 string file = Path.Combine(Backend.obj_dir, Backend.prefix +
122 name.Replace("::",".") + ".ser");
d832c02 @sorear Second draft of serialization code
authored
123 byte[] bytes = File.ReadAllBytes(file);
124
125 su = new SerUnit();
126 su.name = name;
cdb4548 @sorear Checkpoint of global-variable inventory
authored
127 su.hash = NewHash().ComputeHash(bytes);
d832c02 @sorear Second draft of serialization code
authored
128
b25256f @sorear Add Serialize.cs to build, fix build errors
authored
129 ThawBuffer tb = new ThawBuffer(this, su, bytes);
d832c02 @sorear Second draft of serialization code
authored
130
8ed36ea @sorear Wire up serializer to run after pre-compiler
authored
131 units[name] = su;
d832c02 @sorear Second draft of serialization code
authored
132 bool success = false;
133 try {
134 string rsig = tb.String();
135 if (rsig != signature)
136 throw new ThawException("signature mismatch loading " + file);
137 int rver = tb.Int();
138 if (rver != version)
139 throw new ThawException("version mismatch loading " + file);
140
b25256f @sorear Add Serialize.cs to build, fix build errors
authored
141 su.root = tb.ObjRef();
c0af787 @sorear Fix fixup call order (oops!), serialization-compatible calling of pha…
authored
142 tb.RunFixups();
d832c02 @sorear Second draft of serialization code
authored
143 success = true;
144 } finally {
145 // don't leave half-read units in the map
146 if (!success)
147 UnloadUnit(name);
148 }
149
150 return su;
151 }
152
153 // removes a stale unit so a new version can be saved over it.
154 public void UnloadUnit(string name) {
8ed36ea @sorear Wire up serializer to run after pre-compiler
authored
155 if (!units.ContainsKey(name))
156 return;
d832c02 @sorear Second draft of serialization code
authored
157 SerUnit su = units[name];
158 units.Remove(name);
159
160 for (int i = 0; i < su.nobj; i++)
161 byref.Remove(su.bynum[i]);
162 }
163
164 public SerUnit SaveUnit(string name, IFreeze root) {
165 SerUnit su = new SerUnit();
166 su.name = name;
167 su.root = root;
168
169 if (units.ContainsKey(name))
b25256f @sorear Add Serialize.cs to build, fix build errors
authored
170 throw new InvalidOperationException("unit " +name+ " exists");
d832c02 @sorear Second draft of serialization code
authored
171
172 bool success = false;
9083142 @sorear Switch to single-AppDomain execution
authored
173 string file = Path.Combine(Backend.obj_dir, Backend.prefix +
174 name.Replace("::",".") + ".ser");
d832c02 @sorear Second draft of serialization code
authored
175
176 FreezeBuffer fb = new FreezeBuffer(this, su);
177
8ed36ea @sorear Wire up serializer to run after pre-compiler
authored
178 units[name] = su;
179
d832c02 @sorear Second draft of serialization code
authored
180 try {
181 fb.String(signature);
182 fb.Int(version);
183 fb.ObjRef(root);
184
185 byte[] data = fb.GetData();
cdb4548 @sorear Checkpoint of global-variable inventory
authored
186 su.hash = NewHash().ComputeHash(data);
d832c02 @sorear Second draft of serialization code
authored
187 File.WriteAllBytes(file, data);
188 success = true;
189 } finally {
190 if (!success)
191 UnloadUnit(name);
192 }
193
194 return su;
195 }
8184302 @sorear Start draft of serialization/deserialization code
authored
196 }
197
198 // One of these codes is written at the beginning of every object ref
1993029 @sorear Start implementing serialization for our objects
authored
199 enum SerializationCode : byte {
d832c02 @sorear Second draft of serialization code
authored
200 // special
201 Null,
202
8184302 @sorear Start draft of serialization/deserialization code
authored
203 // existing objects
204 ForeignRef,
205 SelfRef,
206 NewUnitRef,
1993029 @sorear Start implementing serialization for our objects
authored
207
208 // types of new object
209 RuntimeUnit,
210 SubInfo,
211 STable,
212 StashEnt,
d91ecdf @sorear Serialization for objects of type Rat, FatRat, Complex, BigInteger, I…
authored
213 Rat,
214 FatRat,
215 Complex,
216 BigInteger,
217 VarDeque,
218 VarHash,
dc6dd55 @sorear Implement serialization for Frame, finally kill off hashtable lexicals
authored
219 DispatchEnt,
220 RxFrame,
221 P6how,
3f80fca @sorear A procedure for serializing ContextHandler et al
authored
222 ReflectObj,
7ae298f @sorear Make CC, LIHint serializable
authored
223 CC,
224 AltInfo,
10ba66d @sorear Turn Signatures into real objects
authored
225 Signature,
226 Parameter,
1993029 @sorear Start implementing serialization for our objects
authored
227
228 // types of P6any-reified object
d91ecdf @sorear Serialization for objects of type Rat, FatRat, Complex, BigInteger, I…
authored
229 P6opaque, // eventually let's specialize this
1993029 @sorear Start implementing serialization for our objects
authored
230 Frame,
231 Cursor,
232
1c5bb04 @sorear Small refactor to AssemblyBuilder setup, start thaw coding
authored
233 // miscellany - keep these in same order as FallbackFreeze
234 String,
235 ArrP6any,
236 ArrVariable,
7ae298f @sorear Make CC, LIHint serializable
authored
237 ArrString,
238 ArrCC,
fe6e595 @sorear Thaw for SimpleVariable, StashEnt
authored
239 Boolean,
1c5bb04 @sorear Small refactor to AssemblyBuilder setup, start thaw coding
authored
240 Int,
241 Double,
3f80fca @sorear A procedure for serializing ContextHandler et al
authored
242 Type,
d91ecdf @sorear Serialization for objects of type Rat, FatRat, Complex, BigInteger, I…
authored
243
ab35238 @sorear Tidy up STable fields a bit, add TiedVariable serialize
authored
244 // variables
1c5bb04 @sorear Small refactor to AssemblyBuilder setup, start thaw coding
authored
245 SimpleVariable, // allow 4 for flags
fe6e595 @sorear Thaw for SimpleVariable, StashEnt
authored
246 SimpleVariable_1,
247 SimpleVariable_2,
248 SimpleVariable_3,
249 SubstrLValue,
ab35238 @sorear Tidy up STable fields a bit, add TiedVariable serialize
authored
250 TiedVariable,
037f4b7 @sorear Make * a "blackhole" container that ignores writes
authored
251 Blackhole,
1993029 @sorear Start implementing serialization for our objects
authored
252
253 // vivification hooks
ab35238 @sorear Tidy up STable fields a bit, add TiedVariable serialize
authored
254 SubViviHook,
1993029 @sorear Start implementing serialization for our objects
authored
255 ArrayViviHook,
256 NewArrayViviHook,
257 HashViviHook,
258 NewHashViviHook,
b1088dc @sorear implement serialization for SubInfo, LAD, LexInfo
authored
259
260 // Longest-token automaton descriptors
261 LADNone, // no-args
262 LADNull,
263 LADDot,
264 LADDispatcher,
265 LADImp,
266 LADStr, // string
267 LADStrNoCase,
268 LADMethod,
269 LADParam,
b8a5ff0 @sorear Treat quantifiers as partially declarative whenever at all possible
authored
270 LADQuant, // byte,LAD,LAD
b1088dc @sorear implement serialization for SubInfo, LAD, LexInfo
authored
271 LADSequence, // LAD[]
272 LADAny,
273 LADCC, // CC
8184302 @sorear Start draft of serialization/deserialization code
authored
274 }
275
276 // An instance of this class is used to serialize serialization units
1993029 @sorear Start implementing serialization for our objects
authored
277 public class FreezeBuffer {
8184302 @sorear Start draft of serialization/deserialization code
authored
278 byte[] data;
279 int wpointer;
280
d832c02 @sorear Second draft of serialization code
authored
281 Dictionary<SerUnit,int> unit_to_offset;
282 int usedunits;
8184302 @sorear Start draft of serialization/deserialization code
authored
283
284 ObjectRegistry reg;
d832c02 @sorear Second draft of serialization code
authored
285 SerUnit unit;
8184302 @sorear Start draft of serialization/deserialization code
authored
286
d832c02 @sorear Second draft of serialization code
authored
287 internal FreezeBuffer(ObjectRegistry reg, SerUnit unit) {
8ed36ea @sorear Wire up serializer to run after pre-compiler
authored
288 if (reg == null || unit == null)
289 throw new ArgumentNullException();
8184302 @sorear Start draft of serialization/deserialization code
authored
290 this.reg = reg;
d832c02 @sorear Second draft of serialization code
authored
291 this.unit = unit;
292 unit_to_offset = new Dictionary<SerUnit,int>();
8184302 @sorear Start draft of serialization/deserialization code
authored
293 data = new byte[256];
294 }
295
b25256f @sorear Add Serialize.cs to build, fix build errors
authored
296 internal byte[] GetData() {
297 byte[] ret = new byte[wpointer];
298 Array.Copy(data, ret, ret.Length);
299 return ret;
300 }
301
8184302 @sorear Start draft of serialization/deserialization code
authored
302 void Ensure(int ct) {
303 while (ct + wpointer > data.Length)
304 Array.Resize(ref data, data.Length * 2);
305 }
306
307 public void Byte(byte x) {
308 Ensure(1);
309 data[wpointer++] = x;
310 }
311
19ada5b @sorear Implement variable-length coding for serialization
authored
312 public void Long(long x) {
313 //Console.WriteLine("Saving {0} at {1}", x, wpointer);
314 Ensure(10);
315 while (true) {
316 if (x >= -64 && x <= 63) {
317 data[wpointer++] = (byte) (127 & (byte)x);
318 break;
319 } else {
320 data[wpointer++] = (byte) (128 | (byte)x);
321 x >>= 7;
322 }
323 }
8184302 @sorear Start draft of serialization/deserialization code
authored
324 }
325
19ada5b @sorear Implement variable-length coding for serialization
authored
326 public void ULong(ulong x) {
327 //Console.WriteLine("Saving {0} at {1}", x, wpointer);
328 Ensure(10);
329 while (true) {
330 if (x <= 127) {
331 data[wpointer++] = (byte) (127 & (byte)x);
332 break;
333 } else {
334 data[wpointer++] = (byte) (128 | (byte)x);
335 x >>= 7;
336 }
337 }
8184302 @sorear Start draft of serialization/deserialization code
authored
338 }
339
19ada5b @sorear Implement variable-length coding for serialization
authored
340 public void Short(short x) { Long(x); }
341 public void Int(int x) { Long(x); }
8184302 @sorear Start draft of serialization/deserialization code
authored
342
8035d26 @sorear Thaw code for VarDeque, VarHash, BigInteger, Complex, Rat, FatRat
authored
343 public void Double(double x) {
344 Long(BitConverter.DoubleToInt64Bits(x));
345 }
346
8184302 @sorear Start draft of serialization/deserialization code
authored
347 public void String(string s) {
348 if (s == null) {
349 Int(-1);
350 } else {
351 Int(s.Length);
352 foreach (char ch in s)
19ada5b @sorear Implement variable-length coding for serialization
authored
353 ULong((ulong)ch);
8184302 @sorear Start draft of serialization/deserialization code
authored
354 }
355 }
356
3dc4789 @sorear Implement freezing for types
authored
357 public void Strings(string[] s) {
358 if (s == null) Int(-1);
359 else {
360 Int(s.Length);
361 foreach (string ch in s) String(ch);
362 }
363 }
364
b1088dc @sorear implement serialization for SubInfo, LAD, LexInfo
authored
365 public void Ints(int[] s) {
366 if (s == null) {
367 Int(-1);
368 } else {
369 Int(s.Length);
370 foreach (int ch in s)
371 Int(ch);
372 }
373 }
374
dc6dd55 @sorear Implement serialization for Frame, finally kill off hashtable lexicals
authored
375 public void Refs<T> (T[] x) {
d91ecdf @sorear Serialization for objects of type Rat, FatRat, Complex, BigInteger, I…
authored
376 if (x == null) {
377 Int(-1);
378 } else {
379 Int(x.Length);
380 foreach (T y in x)
381 ObjRef(y);
382 }
383 }
384
dc6dd55 @sorear Implement serialization for Frame, finally kill off hashtable lexicals
authored
385 public void Refs<T> (IList<T> x) {
3dc4789 @sorear Implement freezing for types
authored
386 if (x == null) {
387 Int(-1);
388 } else {
389 Int(x.Count);
390 foreach (T y in x)
391 ObjRef(y);
392 }
393 }
394
54921c8 @sorear Prevent choking when trying to serialize a multi-dispatch cache
authored
395 public void Ephemeralize() {
396 Byte((byte)SerializationCode.Null);
397 reg.DeleteLastObject(unit);
398 }
399
d832c02 @sorear Second draft of serialization code
authored
400 // This is the main routine you should call from your Freeze
401 // callbacks to freeze an object
d91ecdf @sorear Serialization for objects of type Rat, FatRat, Complex, BigInteger, I…
authored
402 public void ObjRef(object o) {
d832c02 @sorear Second draft of serialization code
authored
403 int id;
404 SerUnit altunit;
19ada5b @sorear Implement variable-length coding for serialization
authored
405 if (Config.SerTrace)
406 Console.WriteLine("Saving {0} at {1}...", o, wpointer);
d832c02 @sorear Second draft of serialization code
authored
407 if (o == null) { // null pointers are special
408 Byte((byte)SerializationCode.Null);
409 return;
410 }
411
412 if (reg.CheckWriteObject(unit, o, out altunit, out id)) {
8184302 @sorear Start draft of serialization/deserialization code
authored
413 if (altunit == unit) {
414 Byte((byte)SerializationCode.SelfRef);
415 } else {
d832c02 @sorear Second draft of serialization code
authored
416 int altcode;
417 if (!unit_to_offset.TryGetValue(altunit, out altcode)) {
8184302 @sorear Start draft of serialization/deserialization code
authored
418 Byte((byte)SerializationCode.NewUnitRef);
b25256f @sorear Add Serialize.cs to build, fix build errors
authored
419 String(altunit.name);
d832c02 @sorear Second draft of serialization code
authored
420 // save the hash too so stale refs can be caught
421 foreach (byte b in altunit.hash) Byte(b);
422
8184302 @sorear Start draft of serialization/deserialization code
authored
423 unit_to_offset[altunit] = usedunits++;
424 } else {
425 Byte((byte)SerializationCode.ForeignRef);
d832c02 @sorear Second draft of serialization code
authored
426 Int(altcode);
8184302 @sorear Start draft of serialization/deserialization code
authored
427 }
428 }
429 Int((int)id);
430 } else {
d832c02 @sorear Second draft of serialization code
authored
431 // must take responsibility for saving the tag
d91ecdf @sorear Serialization for objects of type Rat, FatRat, Complex, BigInteger, I…
authored
432 IFreeze f = o as IFreeze;
433 if (f != null) {
434 f.Freeze(this);
435 } else {
436 FallbackFreeze(o);
437 }
438 }
439 }
440
cdb4548 @sorear Checkpoint of global-variable inventory
authored
441 [Immutable]
be108f9 @sorear Thaw routines for P6opaque, DispatchEnt, double, int, Type, string, b…
authored
442 internal static Type[] boxTypes = new Type[] {
443 null, typeof(Rat), typeof(FatRat), typeof(Complex),
7ae298f @sorear Make CC, LIHint serializable
authored
444 typeof(double), typeof(int), typeof(string), typeof(VarHash),
be108f9 @sorear Thaw routines for P6opaque, DispatchEnt, double, int, Type, string, b…
authored
445 typeof(Variable[]), typeof(VarDeque), typeof(STable),
eba5eb3 @sorear Take a first crack at implementing constant folding
authored
446 typeof(BigInteger),
be108f9 @sorear Thaw routines for P6opaque, DispatchEnt, double, int, Type, string, b…
authored
447 };
cdb4548 @sorear Checkpoint of global-variable inventory
authored
448 [Immutable]
be108f9 @sorear Thaw routines for P6opaque, DispatchEnt, double, int, Type, string, b…
authored
449 internal static Func<P6opaque>[] boxCreate = new Func<P6opaque>[] {
450 P6opaque.Create, BoxObject<Rat>.Create, BoxObject<FatRat>.Create,
451 BoxObject<Complex>.Create, BoxObject<double>.Create,
452 BoxObject<int>.Create, BoxObject<string>.Create,
19ada5b @sorear Implement variable-length coding for serialization
authored
453 BoxObject<VarHash>.Create, BoxObject<Variable[]>.Create,
454 BoxObject<VarDeque>.Create, BoxObject<STable>.Create,
eba5eb3 @sorear Take a first crack at implementing constant folding
authored
455 BoxObject<BigInteger>.Create,
be108f9 @sorear Thaw routines for P6opaque, DispatchEnt, double, int, Type, string, b…
authored
456 };
cdb4548 @sorear Checkpoint of global-variable inventory
authored
457 [Immutable]
d91ecdf @sorear Serialization for objects of type Rat, FatRat, Complex, BigInteger, I…
authored
458 static Type[] anyTypes = new Type[] {
459 typeof(string), typeof(P6any[]), typeof(Variable[]),
7ae298f @sorear Make CC, LIHint serializable
authored
460 typeof(string[]), typeof(CC[]),
3f80fca @sorear A procedure for serializing ContextHandler et al
authored
461 typeof(bool), typeof(int), typeof(double), typeof(Type),
d91ecdf @sorear Serialization for objects of type Rat, FatRat, Complex, BigInteger, I…
authored
462 };
463
464 void FallbackFreeze(object o) {
465 int ix = 0;
466 Type t = o.GetType();
fe6e595 @sorear Thaw for SimpleVariable, StashEnt
authored
467 while (ix != anyTypes.Length && anyTypes[ix] != t) ix++;
1c5bb04 @sorear Small refactor to AssemblyBuilder setup, start thaw coding
authored
468 Byte((byte)(((int)SerializationCode.String) + ix));
d91ecdf @sorear Serialization for objects of type Rat, FatRat, Complex, BigInteger, I…
authored
469
470 switch(ix) {
471 case 0:
472 String((string)o);
473 break;
474 case 1:
475 Refs((P6any[])o);
476 break;
477 case 2:
478 Refs((Variable[])o);
479 break;
480 case 3:
7ae298f @sorear Make CC, LIHint serializable
authored
481 Refs((string[])o);
d91ecdf @sorear Serialization for objects of type Rat, FatRat, Complex, BigInteger, I…
authored
482 break;
483 case 4:
7ae298f @sorear Make CC, LIHint serializable
authored
484 Refs((CC[])o);
fe6e595 @sorear Thaw for SimpleVariable, StashEnt
authored
485 break;
486 case 5:
7ae298f @sorear Make CC, LIHint serializable
authored
487 Byte((byte)((bool)o ? 1 : 0));
d91ecdf @sorear Serialization for objects of type Rat, FatRat, Complex, BigInteger, I…
authored
488 break;
3f80fca @sorear A procedure for serializing ContextHandler et al
authored
489 case 6:
7ae298f @sorear Make CC, LIHint serializable
authored
490 Int((int)o);
491 break;
492 case 7:
493 Double((double)o);
494 break;
495 case 8:
3f80fca @sorear A procedure for serializing ContextHandler et al
authored
496 String(((Type)o).AssemblyQualifiedName);
497 break;
d91ecdf @sorear Serialization for objects of type Rat, FatRat, Complex, BigInteger, I…
authored
498 default:
499 throw new NotImplementedException(t.FullName);
8184302 @sorear Start draft of serialization/deserialization code
authored
500 }
501 }
502 }
503
504 // Note that this interface only handles freezing - thaw is done using
505 // a switch statement.
1993029 @sorear Start implementing serialization for our objects
authored
506 public interface IFreeze {
8184302 @sorear Start draft of serialization/deserialization code
authored
507 void Freeze(FreezeBuffer fb);
508 }
fe6e595 @sorear Thaw for SimpleVariable, StashEnt
authored
509 // implement this if you need to copy in data from other objects, &c
510 interface IFixup {
511 void Fixup();
512 }
8184302 @sorear Start draft of serialization/deserialization code
authored
513
514 class ThawBuffer {
515 byte[] data;
516 int rpointer;
517 ObjectRegistry reg;
518
d832c02 @sorear Second draft of serialization code
authored
519 SerUnit[] unit_map = new SerUnit[8];
8184302 @sorear Start draft of serialization/deserialization code
authored
520 int refed_units;
d832c02 @sorear Second draft of serialization code
authored
521 SerUnit unit;
8184302 @sorear Start draft of serialization/deserialization code
authored
522
fe6e595 @sorear Thaw for SimpleVariable, StashEnt
authored
523 List<IFixup> fixups_needed = new List<IFixup>();
2ff6525 @sorear Thaw routines for types
authored
524 List<object> revalidate = new List<object>();
fe6e595 @sorear Thaw for SimpleVariable, StashEnt
authored
525
288db48 @sorear Use precompiled IL when using a precompiled module
authored
526 public Type type;
5792d5b @sorear Avoid O(n) GetField and GetMethod in thaw procedure
authored
527 public Dictionary<string,System.Reflection.MethodInfo> methods;
288db48 @sorear Use precompiled IL when using a precompiled module
authored
528
b25256f @sorear Add Serialize.cs to build, fix build errors
authored
529 internal ThawBuffer(ObjectRegistry reg, SerUnit unit, byte[] data) {
8184302 @sorear Start draft of serialization/deserialization code
authored
530 this.data = data;
531 this.reg = reg;
d832c02 @sorear Second draft of serialization code
authored
532 this.unit = unit;
8184302 @sorear Start draft of serialization/deserialization code
authored
533 }
534
fe6e595 @sorear Thaw for SimpleVariable, StashEnt
authored
535 internal void RunFixups() {
2ff6525 @sorear Thaw routines for types
authored
536 P6how.BulkRevalidate(revalidate);
fe6e595 @sorear Thaw for SimpleVariable, StashEnt
authored
537 foreach (IFixup f in fixups_needed)
538 f.Fixup();
539 fixups_needed.Clear();
540 }
541
542 internal void PushFixup(IFixup f) {
543 fixups_needed.Add(f);
544 }
545
2ff6525 @sorear Thaw routines for types
authored
546 internal void PushRevalidate(STable f) {
547 revalidate.Add(f);
548 }
549
b25256f @sorear Add Serialize.cs to build, fix build errors
authored
550 public byte Byte() { return data[rpointer++]; }
551
19ada5b @sorear Implement variable-length coding for serialization
authored
552 public long Long() {
553 int shift = 0;
554 long accum = 0;
555 while (true) {
556 byte b = Byte();
557 accum |= (((long)(b & 127)) << shift);
558 shift += 7;
559 if ((b & 128) == 0) {
3a82e24 @sorear Fix deserialization of 64-bit values between -2**62-1 and -2**63
authored
560 if ((b & 64) != 0 && shift < 64) {
19ada5b @sorear Implement variable-length coding for serialization
authored
561 accum |= ((-1L) << shift);
562 }
563 //Console.WriteLine("Read {0} end {1}", accum, rpointer);
564 return accum;
565 }
566 }
b25256f @sorear Add Serialize.cs to build, fix build errors
authored
567 }
568
19ada5b @sorear Implement variable-length coding for serialization
authored
569 public ulong ULong() {
570 int shift = 0;
571 ulong accum = 0;
572 while (true) {
573 byte b = Byte();
574 accum |= (((ulong)(b & 127)) << shift);
575 shift += 7;
576 if ((b & 128) == 0) {
577 //Console.WriteLine("Read {0} end {1}", accum, rpointer);
578 return accum;
579 }
580 }
b25256f @sorear Add Serialize.cs to build, fix build errors
authored
581 }
582
19ada5b @sorear Implement variable-length coding for serialization
authored
583 public short Short() {
584 return checked((short)Long());
585 }
586
587 public int Int() {
588 return checked((int)Long());
b25256f @sorear Add Serialize.cs to build, fix build errors
authored
589 }
590
8035d26 @sorear Thaw code for VarDeque, VarHash, BigInteger, Complex, Rat, FatRat
authored
591 public double Double() {
592 return BitConverter.Int64BitsToDouble(Long());
593 }
594
b25256f @sorear Add Serialize.cs to build, fix build errors
authored
595 public string String() {
596 int l = Int();
597
598 if (l < 0) return null;
599 char[] cb = new char[l];
600
601 for (int i = 0; i < l; i++)
19ada5b @sorear Implement variable-length coding for serialization
authored
602 cb[i] = (char)ULong();
b25256f @sorear Add Serialize.cs to build, fix build errors
authored
603
604 return new string(cb);
605 }
606
607 public byte[] Bytes(int k) {
608 byte[] buf = new byte[k];
609
610 for (int i = 0; i < k; i++)
611 buf[i] = Byte();
612
613 return buf;
614 }
615
1c5bb04 @sorear Small refactor to AssemblyBuilder setup, start thaw coding
authored
616 public List<T> RefsL<T>() where T : class {
617 int ct = Int();
618 if (ct < 0) return null;
619 List<T> ret = new List<T>();
620 for (int i = 0; i < ct; i++)
621 ret.Add((T) ObjRef());
622 return ret;
623 }
624
2ff6525 @sorear Thaw routines for types
authored
625 public int[] Ints() {
626 int ct = Int();
627 if (ct < 0) return null;
628 int[] ret = new int[ct];
629 for (int i = 0; i < ct; i++) ret[i] = Int();
630 return ret;
631 }
632
633 public string[] Strings() {
634 int ct = Int();
635 if (ct < 0) return null;
636 string[] ret = new string[ct];
637 for (int i = 0; i < ct; i++) ret[i] = String();
638 return ret;
639 }
640
1c5bb04 @sorear Small refactor to AssemblyBuilder setup, start thaw coding
authored
641 public T[] RefsA<T>() where T : class {
642 int ct = Int();
643 if (ct < 0) return null;
644 T[] ret = new T[ct];
645 for (int i = 0; i < ct; i++)
646 ret[i] = (T) ObjRef();
647 return ret;
648 }
649
604037a @sorear Fix generic thawing of array objects
authored
650 // used from ObjRef only so guaranteed non-null
651 T[] RefsARegister<T>() where T : class {
652 int ct = Int();
653 T[] ret = new T[ct];
654 Register(ret);
655 for (int i = 0; i < ct; i++)
656 ret[i] = (T) ObjRef();
657 return ret;
658 }
659
b25256f @sorear Add Serialize.cs to build, fix build errors
authored
660 public object ObjRef() {
8184302 @sorear Start draft of serialization/deserialization code
authored
661 var tag = (SerializationCode)Byte();
19ada5b @sorear Implement variable-length coding for serialization
authored
662 if (Config.SerTrace)
663 Console.WriteLine("Reading {0} from {1}...", tag, rpointer-1);
8184302 @sorear Start draft of serialization/deserialization code
authored
664 int i, j;
665 switch(tag) {
d832c02 @sorear Second draft of serialization code
authored
666 case SerializationCode.Null:
667 return null;
8035d26 @sorear Thaw code for VarDeque, VarHash, BigInteger, Complex, Rat, FatRat
authored
668
8184302 @sorear Start draft of serialization/deserialization code
authored
669 case SerializationCode.ForeignRef:
670 i = Int();
671 j = Int();
d832c02 @sorear Second draft of serialization code
authored
672 return unit_map[i].bynum[j];
8035d26 @sorear Thaw code for VarDeque, VarHash, BigInteger, Complex, Rat, FatRat
authored
673 case SerializationCode.SelfRef:
674 i = Int();
675 return unit.bynum[i];
8184302 @sorear Start draft of serialization/deserialization code
authored
676 case SerializationCode.NewUnitRef:
d832c02 @sorear Second draft of serialization code
authored
677 return LoadNewUnit();
8035d26 @sorear Thaw code for VarDeque, VarHash, BigInteger, Complex, Rat, FatRat
authored
678
1c5bb04 @sorear Small refactor to AssemblyBuilder setup, start thaw coding
authored
679 case SerializationCode.RuntimeUnit:
680 return RuntimeUnit.Thaw(this);
2671ab4 @sorear Add thaw routine for SubInfo
authored
681 case SerializationCode.SubInfo:
682 return SubInfo.Thaw(this);
2ff6525 @sorear Thaw routines for types
authored
683 case SerializationCode.STable:
684 return STable.Thaw(this);
fe6e595 @sorear Thaw for SimpleVariable, StashEnt
authored
685 case SerializationCode.StashEnt:
686 return StashEnt.Thaw(this);
8035d26 @sorear Thaw code for VarDeque, VarHash, BigInteger, Complex, Rat, FatRat
authored
687 case SerializationCode.Rat:
688 return Rat.Thaw(this);
689 case SerializationCode.FatRat:
690 return FatRat.Thaw(this);
691 case SerializationCode.Complex:
692 return Complex.Thaw(this);
693 case SerializationCode.BigInteger:
694 return BigInteger.Thaw(this);
695 case SerializationCode.VarDeque:
696 return VarDeque.Thaw(this);
697 case SerializationCode.VarHash:
698 return VarHash.Thaw(this);
be108f9 @sorear Thaw routines for P6opaque, DispatchEnt, double, int, Type, string, b…
authored
699 case SerializationCode.DispatchEnt:
700 return DispatchEnt.Thaw(this);
8035d26 @sorear Thaw code for VarDeque, VarHash, BigInteger, Complex, Rat, FatRat
authored
701 //case SerializationCode.RxFrame:
702 // return RxFrame.Thaw(this);
2ff6525 @sorear Thaw routines for types
authored
703 case SerializationCode.P6how:
704 return P6how.Thaw(this);
7ae298f @sorear Make CC, LIHint serializable
authored
705 case SerializationCode.CC:
706 return CC.Thaw(this);
e7d3f3f @sorear Reimplement regex_infix:<|> table generation
authored
707 case SerializationCode.AltInfo:
708 return AltInfo.Thaw(this);
10ba66d @sorear Turn Signatures into real objects
authored
709 case SerializationCode.Signature:
710 return Signature.Thaw(this);
711 case SerializationCode.Parameter:
712 return Parameter.Thaw(this);
8035d26 @sorear Thaw code for VarDeque, VarHash, BigInteger, Complex, Rat, FatRat
authored
713
3f80fca @sorear A procedure for serializing ContextHandler et al
authored
714 case SerializationCode.ReflectObj:
715 return ReflectObj.Thaw(this);
be108f9 @sorear Thaw routines for P6opaque, DispatchEnt, double, int, Type, string, b…
authored
716 case SerializationCode.P6opaque:
717 return P6opaque.Thaw(this);
9b6504a @sorear Thaw routines for Frame
authored
718 case SerializationCode.Frame:
719 return Frame.Thaw(this);
8035d26 @sorear Thaw code for VarDeque, VarHash, BigInteger, Complex, Rat, FatRat
authored
720 //Cursor,
721
be108f9 @sorear Thaw routines for P6opaque, DispatchEnt, double, int, Type, string, b…
authored
722 case SerializationCode.String:
723 return Register(String());
724 case SerializationCode.ArrP6any:
604037a @sorear Fix generic thawing of array objects
authored
725 return RefsARegister<P6any>();
be108f9 @sorear Thaw routines for P6opaque, DispatchEnt, double, int, Type, string, b…
authored
726 case SerializationCode.ArrVariable:
604037a @sorear Fix generic thawing of array objects
authored
727 return RefsARegister<Variable>();
7ae298f @sorear Make CC, LIHint serializable
authored
728 case SerializationCode.ArrString:
604037a @sorear Fix generic thawing of array objects
authored
729 return RefsARegister<string>();
7ae298f @sorear Make CC, LIHint serializable
authored
730 case SerializationCode.ArrCC:
604037a @sorear Fix generic thawing of array objects
authored
731 return RefsARegister<CC>();
be108f9 @sorear Thaw routines for P6opaque, DispatchEnt, double, int, Type, string, b…
authored
732 case SerializationCode.Boolean:
733 return Register(Byte() != 0);
734 case SerializationCode.Int:
735 return Register(Int());
736 case SerializationCode.Double:
737 return Register(Double());
738 case SerializationCode.Type:
739 return Register(Type.GetType(String(), true));
8035d26 @sorear Thaw code for VarDeque, VarHash, BigInteger, Complex, Rat, FatRat
authored
740
fe6e595 @sorear Thaw for SimpleVariable, StashEnt
authored
741 case SerializationCode.SimpleVariable:
742 case SerializationCode.SimpleVariable_1:
743 case SerializationCode.SimpleVariable_2:
744 case SerializationCode.SimpleVariable_3:
745 return SimpleVariable.Thaw(this,
746 (int)tag - (int)SerializationCode.SimpleVariable);
054561b @sorear Thaw code for all LAD, Variable, ViviHook subclasses
authored
747 case SerializationCode.SubstrLValue:
748 return SubstrLValue.Thaw(this);
749 case SerializationCode.TiedVariable:
750 return TiedVariable.Thaw(this);
037f4b7 @sorear Make * a "blackhole" container that ignores writes
authored
751 case SerializationCode.Blackhole:
752 return Builtins.Blackhole.Thaw(this);
054561b @sorear Thaw code for all LAD, Variable, ViviHook subclasses
authored
753
754 case SerializationCode.SubViviHook:
755 return SubViviHook.Thaw(this);
756 case SerializationCode.ArrayViviHook:
757 return ArrayViviHook.Thaw(this);
758 case SerializationCode.NewArrayViviHook:
759 return NewArrayViviHook.Thaw(this);
760 case SerializationCode.HashViviHook:
761 return HashViviHook.Thaw(this);
762 case SerializationCode.NewHashViviHook:
763 return NewHashViviHook.Thaw(this);
764
765 case SerializationCode.LADNone:
766 return Register(new LADNone());
767 case SerializationCode.LADNull:
768 return Register(new LADNull());
769 case SerializationCode.LADDot:
770 return Register(new LADDot());
771 case SerializationCode.LADDispatcher:
772 return Register(new LADDispatcher());
773 case SerializationCode.LADImp:
774 return Register(new LADImp());
775 case SerializationCode.LADStr:
776 return LADStr.Thaw(this);
777 case SerializationCode.LADStrNoCase:
778 return LADStrNoCase.Thaw(this);
779 case SerializationCode.LADMethod:
780 return LADMethod.Thaw(this);
781 case SerializationCode.LADParam:
782 return LADParam.Thaw(this);
b8a5ff0 @sorear Treat quantifiers as partially declarative whenever at all possible
authored
783 case SerializationCode.LADQuant:
784 return LADQuant.Thaw(this);
054561b @sorear Thaw code for all LAD, Variable, ViviHook subclasses
authored
785 case SerializationCode.LADSequence:
786 return LADSequence.Thaw(this);
787 case SerializationCode.LADAny:
788 return LADAny.Thaw(this);
789 case SerializationCode.LADCC:
790 return LADCC.Thaw(this);
8184302 @sorear Start draft of serialization/deserialization code
authored
791 default:
1c5bb04 @sorear Small refactor to AssemblyBuilder setup, start thaw coding
authored
792 throw new ThawException("unexpected object tag " + tag);
8184302 @sorear Start draft of serialization/deserialization code
authored
793 }
794 }
d832c02 @sorear Second draft of serialization code
authored
795
1c5bb04 @sorear Small refactor to AssemblyBuilder setup, start thaw coding
authored
796 // call this when thawing any new object
be108f9 @sorear Thaw routines for P6opaque, DispatchEnt, double, int, Type, string, b…
authored
797 internal object Register(object o) {
1c5bb04 @sorear Small refactor to AssemblyBuilder setup, start thaw coding
authored
798 reg.RegisterThawed(unit, o);
be108f9 @sorear Thaw routines for P6opaque, DispatchEnt, double, int, Type, string, b…
authored
799 return o;
1c5bb04 @sorear Small refactor to AssemblyBuilder setup, start thaw coding
authored
800 }
801
d832c02 @sorear Second draft of serialization code
authored
802 object LoadNewUnit() {
803 string name = String();
804 if (refed_units == unit_map.Length)
805 Array.Resize(ref unit_map, refed_units * 2);
806
807 SerUnit su = reg.LoadUnit(name);
808 unit_map[refed_units++] = su;
809
810 byte[] hash = Bytes(su.hash.Length);
811
812 for (int i = 0; i < hash.Length; i++)
813 if (hash[i] != su.hash[i])
814 goto badhash;
815
816 int ix = Int();
817 return su.bynum[ix];
818
819 badhash:
eee0cbb @sorear Add framework for detecting stale precompiled modules
authored
820 throw new ThawException(string.Format("Hash mismatch for " +
821 "unit {0} referenced from {1}, wanted {2}, got {3}",
822 su.name, unit.name, Utils.HashToStr(unit.hash),
823 Utils.HashToStr(su.hash)));
d832c02 @sorear Second draft of serialization code
authored
824 }
825 }
826
827 // Thrown to indicate data format problems in the serialized stream
828 // Not necessarily bugs; could also indicate stale files, including
829 // cases where the data format is changed and cases where a depended
830 // file was recreated
831 class ThawException : Exception {
832 public ThawException(string s) : base(s) { }
833 public ThawException() : base() { }
8184302 @sorear Start draft of serialization/deserialization code
authored
834 }
3f80fca @sorear A procedure for serializing ContextHandler et al
authored
835
836 public class ReflectObj : IFreeze {
837 protected virtual object[] GetData() { return new object[0]; }
838 protected virtual void SetData(object[] a) { }
839 void IFreeze.Freeze(FreezeBuffer fb) {
840 fb.Byte((byte)SerializationCode.ReflectObj);
841 fb.String(GetType().AssemblyQualifiedName);
842 fb.Refs(GetData());
843 }
844
845 internal static ReflectObj Thaw(ThawBuffer tb) {
ead51d9 @sorear Skeleton of -gen-app; use named methods for builtin methods to reduce…
authored
846 string nm = tb.String();
847 if (Backend.cross_level_load)
848 nm = nm.Replace("Run.", "");
849 Type nt = Type.GetType(nm, true);
3f80fca @sorear A procedure for serializing ContextHandler et al
authored
850 ReflectObj n = (ReflectObj)
f0788e8 @sorear Small bugfixes, use correct form of Invoke for .net
authored
851 nt.GetConstructor(new Type[0]).Invoke(new object[0]);
3f80fca @sorear A procedure for serializing ContextHandler et al
authored
852 tb.Register(n);
853 n.SetData(tb.RefsA<object>());
854 return n;
855 }
856 }
8184302 @sorear Start draft of serialization/deserialization code
authored
857 }
Something went wrong with that request. Please try again.