Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Newer
Older
100644 861 lines (747 sloc) 28.286 kb
8184302 @sorear Start draft of serialization/deserialization code
authored
1 using System;
d832c02 @sorear Second draft of serialization code
authored
2 using System.IO;
3 using System.Security.Cryptography;
b25256f @sorear Add Serialize.cs to build, fix build errors
authored
4 using System.Collections.Generic;
ee02ddf @sorear Tweaks to allow saving and thawing cross-unit deep sub clones, such as w...
authored
5 using System.Reflection;
d832c02 @sorear Second draft of serialization code
authored
6 using System.Text;
9083142 @sorear Switch to single-AppDomain execution
authored
7 using Niecza.CLRBackend;
8184302 @sorear Start draft of serialization/deserialization code
authored
8
cdde7e0 @sorear Notes on four kinds of module scope
authored
9 // Here in Niecza we have four different kinds of unit scopes:
10 //
11 // * COMPILING::UNIT, aka RuntimeUnit: one of these exists for every
12 // call into the compiler, whether eval or module. The REPL will
13 // be considered as if it were eval.
14 //
15 // * Serialization scopes, which are created when compiling modules
16 // or when pre-compiling a main program. During execution there is
17 // no current serialization scope; evals inherit the serialization
18 // scope or lack thereof that was in effect.
19 //
20 // * Assembly scopes, which are an artifact of the CLR and almost align
21 // with precompilation scopes, except that they have to exist always
22 // because methods cannot be created free-floating in CLR 2.x.
23 //
24 // An assembly scope is created for all serialization scopes, and
25 // non-saved anonymous assemblies are created for eval-and-run of
26 // a file and non-BEGIN-time evals.
27 //
28 // * GLOBAL scope is very much like serialization scope except that there
29 // is a "true globals" scope that is used when not serializing.
30
8184302 @sorear Start draft of serialization/deserialization code
authored
31 // This implements "bounded serialization" for Niecza. Unfortunately
32 // the CLR's builtin serialization can't efficiently be made bounded,
33 // and anyway it would be nice if the serialization format could be
34 // transported across backends.
35
36 // TODO: implement a more Storable-like interface.
d832c02 @sorear Second draft of serialization code
authored
37
38 // Note, the serialization subsystem is *NOT* thread safe !
8184302 @sorear Start draft of serialization/deserialization code
authored
39 namespace Niecza.Serialization {
d832c02 @sorear Second draft of serialization code
authored
40 // Information kept on a serialization unit after loading or storing,
41 // but not before storing.
42 class SerUnit {
43 internal string name; // eg "File.Copy"
44 internal byte[] hash; // hash of entire file, filled at write time
8ed36ea @sorear Wire up serializer to run after pre-compiler
authored
45 internal object[] bynum = new object[8]; // objects in unit
d832c02 @sorear Second draft of serialization code
authored
46 internal object root; // the RuntimeUnit object
8ed36ea @sorear Wire up serializer to run after pre-compiler
authored
47 internal int nobj; // = 0
d832c02 @sorear Second draft of serialization code
authored
48 }
8184302 @sorear Start draft of serialization/deserialization code
authored
49
50 // The central feature of *bounded* serialization is that object
51 // registries are kept distinct from the (de)serializer, and can
52 // be shared between serialization runs.
53 class ObjectRegistry {
d832c02 @sorear Second draft of serialization code
authored
54 // TODO: investigate a more specialized representation,
55 // ideally not having to hash as many objects
56 struct ObjRef {
b25256f @sorear Add Serialize.cs to build, fix build errors
authored
57 public SerUnit unit;
58 public int id;
8184302 @sorear Start draft of serialization/deserialization code
authored
59 }
d832c02 @sorear Second draft of serialization code
authored
60 Dictionary<object,ObjRef> byref = new Dictionary<object,ObjRef>();
8184302 @sorear Start draft of serialization/deserialization code
authored
61
d832c02 @sorear Second draft of serialization code
authored
62 Dictionary<string,SerUnit> units =
63 new Dictionary<string,SerUnit>();
64
cdb4548 @sorear Checkpoint of global-variable inventory
authored
65 internal static HashAlgorithm NewHash() { return new SHA256Managed(); }
66
ee02ddf @sorear Tweaks to allow saving and thawing cross-unit deep sub clones, such as w...
authored
67 internal Dictionary<string,Dictionary<string,MethodInfo>> methods =
68 new Dictionary<string,Dictionary<string,MethodInfo>>();
69
d832c02 @sorear Second draft of serialization code
authored
70 static readonly string signature = "Niecza-Serialized-Module";
ee02ddf @sorear Tweaks to allow saving and thawing cross-unit deep sub clones, such as w...
authored
71 static readonly int version = 24;
d832c02 @sorear Second draft of serialization code
authored
72
73 // Routines for use by serialization code
74 public bool CheckWriteObject(SerUnit into, object o,
75 out SerUnit lui, out int id) {
76 ObjRef or;
77 if (byref.TryGetValue(o, out or)) {
78 lui = or.unit;
79 id = or.id;
8184302 @sorear Start draft of serialization/deserialization code
authored
80 return true;
d832c02 @sorear Second draft of serialization code
authored
81 }
8184302 @sorear Start draft of serialization/deserialization code
authored
82
d832c02 @sorear Second draft of serialization code
authored
83 if (into.nobj == into.bynum.Length)
84 Array.Resize(ref into.bynum, into.nobj * 2);
8184302 @sorear Start draft of serialization/deserialization code
authored
85
b25256f @sorear Add Serialize.cs to build, fix build errors
authored
86 or.unit = lui = into;
d832c02 @sorear Second draft of serialization code
authored
87 id = or.id = into.nobj++;
88 into.bynum[id] = o;
89
90 byref[o] = or;
8184302 @sorear Start draft of serialization/deserialization code
authored
91
92 return false;
93 }
d832c02 @sorear Second draft of serialization code
authored
94
54921c8 @sorear Prevent choking when trying to serialize a multi-dispatch cache
authored
95 // Undo previous method...
96 public void DeleteLastObject(SerUnit into) {
97 byref.Remove(into.bynum[--into.nobj]);
98 into.bynum[into.nobj] = null;
99 }
100
1c5bb04 @sorear Small refactor to AssemblyBuilder setup, start thaw coding
authored
101 public void RegisterThawed(SerUnit into, object o) {
102 ObjRef or;
103 if (into.nobj == into.bynum.Length)
104 Array.Resize(ref into.bynum, into.nobj * 2);
105
106 or.unit = into;
107 or.id = into.nobj++;
108 into.bynum[or.id] = o;
109
110 byref[o] = or;
111 }
112
d832c02 @sorear Second draft of serialization code
authored
113 // Routines for use by compilation manager
114
115 // Loads a single unit from the compiled-data directory.
116 // Will throw a ThawException if a stale reference is encountered
117 // or other data format error.
118 public SerUnit LoadUnit(string name) {
119 SerUnit su;
120
121 // is the unit already loaded?
122 if (units.TryGetValue(name, out su))
123 return su;
124
9083142 @sorear Switch to single-AppDomain execution
authored
125 string file = Path.Combine(Backend.obj_dir, Backend.prefix +
126 name.Replace("::",".") + ".ser");
d832c02 @sorear Second draft of serialization code
authored
127 byte[] bytes = File.ReadAllBytes(file);
128
129 su = new SerUnit();
130 su.name = name;
cdb4548 @sorear Checkpoint of global-variable inventory
authored
131 su.hash = NewHash().ComputeHash(bytes);
d832c02 @sorear Second draft of serialization code
authored
132
b25256f @sorear Add Serialize.cs to build, fix build errors
authored
133 ThawBuffer tb = new ThawBuffer(this, su, bytes);
d832c02 @sorear Second draft of serialization code
authored
134
8ed36ea @sorear Wire up serializer to run after pre-compiler
authored
135 units[name] = su;
d832c02 @sorear Second draft of serialization code
authored
136 bool success = false;
137 try {
138 string rsig = tb.String();
139 if (rsig != signature)
140 throw new ThawException("signature mismatch loading " + file);
141 int rver = tb.Int();
142 if (rver != version)
143 throw new ThawException("version mismatch loading " + file);
144
b25256f @sorear Add Serialize.cs to build, fix build errors
authored
145 su.root = tb.ObjRef();
c0af787 @sorear Fix fixup call order (oops!), serialization-compatible calling of phaser...
authored
146 tb.RunFixups();
d832c02 @sorear Second draft of serialization code
authored
147 success = true;
148 } finally {
149 // don't leave half-read units in the map
150 if (!success)
151 UnloadUnit(name);
152 }
153
154 return su;
155 }
156
157 // removes a stale unit so a new version can be saved over it.
158 public void UnloadUnit(string name) {
8ed36ea @sorear Wire up serializer to run after pre-compiler
authored
159 if (!units.ContainsKey(name))
160 return;
d832c02 @sorear Second draft of serialization code
authored
161 SerUnit su = units[name];
162 units.Remove(name);
163
164 for (int i = 0; i < su.nobj; i++)
165 byref.Remove(su.bynum[i]);
166 }
167
168 public SerUnit SaveUnit(string name, IFreeze root) {
169 SerUnit su = new SerUnit();
170 su.name = name;
171 su.root = root;
172
173 if (units.ContainsKey(name))
b25256f @sorear Add Serialize.cs to build, fix build errors
authored
174 throw new InvalidOperationException("unit " +name+ " exists");
d832c02 @sorear Second draft of serialization code
authored
175
176 bool success = false;
9083142 @sorear Switch to single-AppDomain execution
authored
177 string file = Path.Combine(Backend.obj_dir, Backend.prefix +
178 name.Replace("::",".") + ".ser");
d832c02 @sorear Second draft of serialization code
authored
179
180 FreezeBuffer fb = new FreezeBuffer(this, su);
181
8ed36ea @sorear Wire up serializer to run after pre-compiler
authored
182 units[name] = su;
183
d832c02 @sorear Second draft of serialization code
authored
184 try {
185 fb.String(signature);
186 fb.Int(version);
187 fb.ObjRef(root);
188
189 byte[] data = fb.GetData();
cdb4548 @sorear Checkpoint of global-variable inventory
authored
190 su.hash = NewHash().ComputeHash(data);
d832c02 @sorear Second draft of serialization code
authored
191 File.WriteAllBytes(file, data);
192 success = true;
193 } finally {
194 if (!success)
195 UnloadUnit(name);
196 }
197
198 return su;
199 }
8184302 @sorear Start draft of serialization/deserialization code
authored
200 }
201
202 // One of these codes is written at the beginning of every object ref
1993029 @sorear Start implementing serialization for our objects
authored
203 enum SerializationCode : byte {
d832c02 @sorear Second draft of serialization code
authored
204 // special
205 Null,
206
8184302 @sorear Start draft of serialization/deserialization code
authored
207 // existing objects
208 ForeignRef,
209 SelfRef,
210 NewUnitRef,
1993029 @sorear Start implementing serialization for our objects
authored
211
212 // types of new object
213 RuntimeUnit,
214 SubInfo,
215 STable,
216 StashEnt,
d91ecdf @sorear Serialization for objects of type Rat, FatRat, Complex, BigInteger, Int3...
authored
217 Rat,
218 FatRat,
219 Complex,
220 BigInteger,
221 VarDeque,
222 VarHash,
dc6dd55 @sorear Implement serialization for Frame, finally kill off hashtable lexicals
authored
223 DispatchEnt,
224 RxFrame,
225 P6how,
3f80fca @sorear A procedure for serializing ContextHandler et al
authored
226 ReflectObj,
7ae298f @sorear Make CC, LIHint serializable
authored
227 CC,
228 AltInfo,
10ba66d @sorear Turn Signatures into real objects
authored
229 Signature,
230 Parameter,
1993029 @sorear Start implementing serialization for our objects
authored
231
232 // types of P6any-reified object
d91ecdf @sorear Serialization for objects of type Rat, FatRat, Complex, BigInteger, Int3...
authored
233 P6opaque, // eventually let's specialize this
1993029 @sorear Start implementing serialization for our objects
authored
234 Frame,
235 Cursor,
236
1c5bb04 @sorear Small refactor to AssemblyBuilder setup, start thaw coding
authored
237 // miscellany - keep these in same order as FallbackFreeze
238 String,
239 ArrP6any,
240 ArrVariable,
7ae298f @sorear Make CC, LIHint serializable
authored
241 ArrString,
242 ArrCC,
fe6e595 @sorear Thaw for SimpleVariable, StashEnt
authored
243 Boolean,
1c5bb04 @sorear Small refactor to AssemblyBuilder setup, start thaw coding
authored
244 Int,
245 Double,
3f80fca @sorear A procedure for serializing ContextHandler et al
authored
246 Type,
d91ecdf @sorear Serialization for objects of type Rat, FatRat, Complex, BigInteger, Int3...
authored
247
ab35238 @sorear Tidy up STable fields a bit, add TiedVariable serialize
authored
248 // variables
1c5bb04 @sorear Small refactor to AssemblyBuilder setup, start thaw coding
authored
249 SimpleVariable, // allow 4 for flags
fe6e595 @sorear Thaw for SimpleVariable, StashEnt
authored
250 SimpleVariable_1,
251 SimpleVariable_2,
252 SimpleVariable_3,
253 SubstrLValue,
ab35238 @sorear Tidy up STable fields a bit, add TiedVariable serialize
authored
254 TiedVariable,
037f4b7 @sorear Make * a "blackhole" container that ignores writes
authored
255 Blackhole,
1993029 @sorear Start implementing serialization for our objects
authored
256
257 // vivification hooks
ab35238 @sorear Tidy up STable fields a bit, add TiedVariable serialize
authored
258 SubViviHook,
1993029 @sorear Start implementing serialization for our objects
authored
259 ArrayViviHook,
260 NewArrayViviHook,
261 HashViviHook,
262 NewHashViviHook,
b1088dc @sorear implement serialization for SubInfo, LAD, LexInfo
authored
263
264 // Longest-token automaton descriptors
265 LADNone, // no-args
266 LADNull,
267 LADDot,
268 LADDispatcher,
269 LADImp,
270 LADStr, // string
271 LADStrNoCase,
272 LADMethod,
273 LADParam,
b8a5ff0 @sorear Treat quantifiers as partially declarative whenever at all possible
authored
274 LADQuant, // byte,LAD,LAD
b1088dc @sorear implement serialization for SubInfo, LAD, LexInfo
authored
275 LADSequence, // LAD[]
276 LADAny,
277 LADCC, // CC
8184302 @sorear Start draft of serialization/deserialization code
authored
278 }
279
280 // An instance of this class is used to serialize serialization units
1993029 @sorear Start implementing serialization for our objects
authored
281 public class FreezeBuffer {
8184302 @sorear Start draft of serialization/deserialization code
authored
282 byte[] data;
283 int wpointer;
284
d832c02 @sorear Second draft of serialization code
authored
285 Dictionary<SerUnit,int> unit_to_offset;
286 int usedunits;
8184302 @sorear Start draft of serialization/deserialization code
authored
287
288 ObjectRegistry reg;
d832c02 @sorear Second draft of serialization code
authored
289 SerUnit unit;
8184302 @sorear Start draft of serialization/deserialization code
authored
290
d832c02 @sorear Second draft of serialization code
authored
291 internal FreezeBuffer(ObjectRegistry reg, SerUnit unit) {
8ed36ea @sorear Wire up serializer to run after pre-compiler
authored
292 if (reg == null || unit == null)
293 throw new ArgumentNullException();
8184302 @sorear Start draft of serialization/deserialization code
authored
294 this.reg = reg;
d832c02 @sorear Second draft of serialization code
authored
295 this.unit = unit;
296 unit_to_offset = new Dictionary<SerUnit,int>();
8184302 @sorear Start draft of serialization/deserialization code
authored
297 data = new byte[256];
298 }
299
b25256f @sorear Add Serialize.cs to build, fix build errors
authored
300 internal byte[] GetData() {
301 byte[] ret = new byte[wpointer];
302 Array.Copy(data, ret, ret.Length);
303 return ret;
304 }
305
8184302 @sorear Start draft of serialization/deserialization code
authored
306 void Ensure(int ct) {
307 while (ct + wpointer > data.Length)
308 Array.Resize(ref data, data.Length * 2);
309 }
310
311 public void Byte(byte x) {
312 Ensure(1);
313 data[wpointer++] = x;
314 }
315
19ada5b @sorear Implement variable-length coding for serialization
authored
316 public void Long(long x) {
317 //Console.WriteLine("Saving {0} at {1}", x, wpointer);
318 Ensure(10);
319 while (true) {
320 if (x >= -64 && x <= 63) {
321 data[wpointer++] = (byte) (127 & (byte)x);
322 break;
323 } else {
324 data[wpointer++] = (byte) (128 | (byte)x);
325 x >>= 7;
326 }
327 }
8184302 @sorear Start draft of serialization/deserialization code
authored
328 }
329
19ada5b @sorear Implement variable-length coding for serialization
authored
330 public void ULong(ulong x) {
331 //Console.WriteLine("Saving {0} at {1}", x, wpointer);
332 Ensure(10);
333 while (true) {
334 if (x <= 127) {
335 data[wpointer++] = (byte) (127 & (byte)x);
336 break;
337 } else {
338 data[wpointer++] = (byte) (128 | (byte)x);
339 x >>= 7;
340 }
341 }
8184302 @sorear Start draft of serialization/deserialization code
authored
342 }
343
19ada5b @sorear Implement variable-length coding for serialization
authored
344 public void Short(short x) { Long(x); }
345 public void Int(int x) { Long(x); }
8184302 @sorear Start draft of serialization/deserialization code
authored
346
8035d26 @sorear Thaw code for VarDeque, VarHash, BigInteger, Complex, Rat, FatRat
authored
347 public void Double(double x) {
348 Long(BitConverter.DoubleToInt64Bits(x));
349 }
350
8184302 @sorear Start draft of serialization/deserialization code
authored
351 public void String(string s) {
352 if (s == null) {
353 Int(-1);
354 } else {
355 Int(s.Length);
356 foreach (char ch in s)
19ada5b @sorear Implement variable-length coding for serialization
authored
357 ULong((ulong)ch);
8184302 @sorear Start draft of serialization/deserialization code
authored
358 }
359 }
360
3dc4789 @sorear Implement freezing for types
authored
361 public void Strings(string[] s) {
362 if (s == null) Int(-1);
363 else {
364 Int(s.Length);
365 foreach (string ch in s) String(ch);
366 }
367 }
368
b1088dc @sorear implement serialization for SubInfo, LAD, LexInfo
authored
369 public void Ints(int[] s) {
370 if (s == null) {
371 Int(-1);
372 } else {
373 Int(s.Length);
374 foreach (int ch in s)
375 Int(ch);
376 }
377 }
378
dc6dd55 @sorear Implement serialization for Frame, finally kill off hashtable lexicals
authored
379 public void Refs<T> (T[] x) {
d91ecdf @sorear Serialization for objects of type Rat, FatRat, Complex, BigInteger, Int3...
authored
380 if (x == null) {
381 Int(-1);
382 } else {
383 Int(x.Length);
384 foreach (T y in x)
385 ObjRef(y);
386 }
387 }
388
dc6dd55 @sorear Implement serialization for Frame, finally kill off hashtable lexicals
authored
389 public void Refs<T> (IList<T> x) {
3dc4789 @sorear Implement freezing for types
authored
390 if (x == null) {
391 Int(-1);
392 } else {
393 Int(x.Count);
394 foreach (T y in x)
395 ObjRef(y);
396 }
397 }
398
54921c8 @sorear Prevent choking when trying to serialize a multi-dispatch cache
authored
399 public void Ephemeralize() {
400 Byte((byte)SerializationCode.Null);
401 reg.DeleteLastObject(unit);
402 }
403
d832c02 @sorear Second draft of serialization code
authored
404 // This is the main routine you should call from your Freeze
405 // callbacks to freeze an object
d91ecdf @sorear Serialization for objects of type Rat, FatRat, Complex, BigInteger, Int3...
authored
406 public void ObjRef(object o) {
d832c02 @sorear Second draft of serialization code
authored
407 int id;
408 SerUnit altunit;
19ada5b @sorear Implement variable-length coding for serialization
authored
409 if (Config.SerTrace)
410 Console.WriteLine("Saving {0} at {1}...", o, wpointer);
d832c02 @sorear Second draft of serialization code
authored
411 if (o == null) { // null pointers are special
412 Byte((byte)SerializationCode.Null);
413 return;
414 }
415
416 if (reg.CheckWriteObject(unit, o, out altunit, out id)) {
8184302 @sorear Start draft of serialization/deserialization code
authored
417 if (altunit == unit) {
418 Byte((byte)SerializationCode.SelfRef);
419 } else {
d832c02 @sorear Second draft of serialization code
authored
420 int altcode;
421 if (!unit_to_offset.TryGetValue(altunit, out altcode)) {
8184302 @sorear Start draft of serialization/deserialization code
authored
422 Byte((byte)SerializationCode.NewUnitRef);
b25256f @sorear Add Serialize.cs to build, fix build errors
authored
423 String(altunit.name);
d832c02 @sorear Second draft of serialization code
authored
424 // save the hash too so stale refs can be caught
425 foreach (byte b in altunit.hash) Byte(b);
426
8184302 @sorear Start draft of serialization/deserialization code
authored
427 unit_to_offset[altunit] = usedunits++;
428 } else {
429 Byte((byte)SerializationCode.ForeignRef);
d832c02 @sorear Second draft of serialization code
authored
430 Int(altcode);
8184302 @sorear Start draft of serialization/deserialization code
authored
431 }
432 }
433 Int((int)id);
434 } else {
d832c02 @sorear Second draft of serialization code
authored
435 // must take responsibility for saving the tag
d91ecdf @sorear Serialization for objects of type Rat, FatRat, Complex, BigInteger, Int3...
authored
436 IFreeze f = o as IFreeze;
437 if (f != null) {
438 f.Freeze(this);
439 } else {
440 FallbackFreeze(o);
441 }
442 }
443 }
444
cdb4548 @sorear Checkpoint of global-variable inventory
authored
445 [Immutable]
be108f9 @sorear Thaw routines for P6opaque, DispatchEnt, double, int, Type, string, bool...
authored
446 internal static Type[] boxTypes = new Type[] {
447 null, typeof(Rat), typeof(FatRat), typeof(Complex),
7ae298f @sorear Make CC, LIHint serializable
authored
448 typeof(double), typeof(int), typeof(string), typeof(VarHash),
be108f9 @sorear Thaw routines for P6opaque, DispatchEnt, double, int, Type, string, bool...
authored
449 typeof(Variable[]), typeof(VarDeque), typeof(STable),
eba5eb3 @sorear Take a first crack at implementing constant folding
authored
450 typeof(BigInteger),
be108f9 @sorear Thaw routines for P6opaque, DispatchEnt, double, int, Type, string, bool...
authored
451 };
cdb4548 @sorear Checkpoint of global-variable inventory
authored
452 [Immutable]
be108f9 @sorear Thaw routines for P6opaque, DispatchEnt, double, int, Type, string, bool...
authored
453 internal static Func<P6opaque>[] boxCreate = new Func<P6opaque>[] {
454 P6opaque.Create, BoxObject<Rat>.Create, BoxObject<FatRat>.Create,
455 BoxObject<Complex>.Create, BoxObject<double>.Create,
456 BoxObject<int>.Create, BoxObject<string>.Create,
19ada5b @sorear Implement variable-length coding for serialization
authored
457 BoxObject<VarHash>.Create, BoxObject<Variable[]>.Create,
458 BoxObject<VarDeque>.Create, BoxObject<STable>.Create,
eba5eb3 @sorear Take a first crack at implementing constant folding
authored
459 BoxObject<BigInteger>.Create,
be108f9 @sorear Thaw routines for P6opaque, DispatchEnt, double, int, Type, string, bool...
authored
460 };
cdb4548 @sorear Checkpoint of global-variable inventory
authored
461 [Immutable]
d91ecdf @sorear Serialization for objects of type Rat, FatRat, Complex, BigInteger, Int3...
authored
462 static Type[] anyTypes = new Type[] {
463 typeof(string), typeof(P6any[]), typeof(Variable[]),
7ae298f @sorear Make CC, LIHint serializable
authored
464 typeof(string[]), typeof(CC[]),
3f80fca @sorear A procedure for serializing ContextHandler et al
authored
465 typeof(bool), typeof(int), typeof(double), typeof(Type),
d91ecdf @sorear Serialization for objects of type Rat, FatRat, Complex, BigInteger, Int3...
authored
466 };
467
468 void FallbackFreeze(object o) {
469 int ix = 0;
470 Type t = o.GetType();
fe6e595 @sorear Thaw for SimpleVariable, StashEnt
authored
471 while (ix != anyTypes.Length && anyTypes[ix] != t) ix++;
1c5bb04 @sorear Small refactor to AssemblyBuilder setup, start thaw coding
authored
472 Byte((byte)(((int)SerializationCode.String) + ix));
d91ecdf @sorear Serialization for objects of type Rat, FatRat, Complex, BigInteger, Int3...
authored
473
474 switch(ix) {
475 case 0:
476 String((string)o);
477 break;
478 case 1:
479 Refs((P6any[])o);
480 break;
481 case 2:
482 Refs((Variable[])o);
483 break;
484 case 3:
7ae298f @sorear Make CC, LIHint serializable
authored
485 Refs((string[])o);
d91ecdf @sorear Serialization for objects of type Rat, FatRat, Complex, BigInteger, Int3...
authored
486 break;
487 case 4:
7ae298f @sorear Make CC, LIHint serializable
authored
488 Refs((CC[])o);
fe6e595 @sorear Thaw for SimpleVariable, StashEnt
authored
489 break;
490 case 5:
7ae298f @sorear Make CC, LIHint serializable
authored
491 Byte((byte)((bool)o ? 1 : 0));
d91ecdf @sorear Serialization for objects of type Rat, FatRat, Complex, BigInteger, Int3...
authored
492 break;
3f80fca @sorear A procedure for serializing ContextHandler et al
authored
493 case 6:
7ae298f @sorear Make CC, LIHint serializable
authored
494 Int((int)o);
495 break;
496 case 7:
497 Double((double)o);
498 break;
499 case 8:
3f80fca @sorear A procedure for serializing ContextHandler et al
authored
500 String(((Type)o).AssemblyQualifiedName);
501 break;
d91ecdf @sorear Serialization for objects of type Rat, FatRat, Complex, BigInteger, Int3...
authored
502 default:
503 throw new NotImplementedException(t.FullName);
8184302 @sorear Start draft of serialization/deserialization code
authored
504 }
505 }
506 }
507
508 // Note that this interface only handles freezing - thaw is done using
509 // a switch statement.
1993029 @sorear Start implementing serialization for our objects
authored
510 public interface IFreeze {
8184302 @sorear Start draft of serialization/deserialization code
authored
511 void Freeze(FreezeBuffer fb);
512 }
fe6e595 @sorear Thaw for SimpleVariable, StashEnt
authored
513 // implement this if you need to copy in data from other objects, &c
514 interface IFixup {
515 void Fixup();
516 }
8184302 @sorear Start draft of serialization/deserialization code
authored
517
518 class ThawBuffer {
519 byte[] data;
520 int rpointer;
521 ObjectRegistry reg;
522
d832c02 @sorear Second draft of serialization code
authored
523 SerUnit[] unit_map = new SerUnit[8];
8184302 @sorear Start draft of serialization/deserialization code
authored
524 int refed_units;
d832c02 @sorear Second draft of serialization code
authored
525 SerUnit unit;
8184302 @sorear Start draft of serialization/deserialization code
authored
526
fe6e595 @sorear Thaw for SimpleVariable, StashEnt
authored
527 List<IFixup> fixups_needed = new List<IFixup>();
2ff6525 @sorear Thaw routines for types
authored
528 List<object> revalidate = new List<object>();
fe6e595 @sorear Thaw for SimpleVariable, StashEnt
authored
529
288db48 @sorear Use precompiled IL when using a precompiled module
authored
530 public Type type;
531
b25256f @sorear Add Serialize.cs to build, fix build errors
authored
532 internal ThawBuffer(ObjectRegistry reg, SerUnit unit, byte[] data) {
8184302 @sorear Start draft of serialization/deserialization code
authored
533 this.data = data;
534 this.reg = reg;
d832c02 @sorear Second draft of serialization code
authored
535 this.unit = unit;
8184302 @sorear Start draft of serialization/deserialization code
authored
536 }
537
fe6e595 @sorear Thaw for SimpleVariable, StashEnt
authored
538 internal void RunFixups() {
2ff6525 @sorear Thaw routines for types
authored
539 P6how.BulkRevalidate(revalidate);
fe6e595 @sorear Thaw for SimpleVariable, StashEnt
authored
540 foreach (IFixup f in fixups_needed)
541 f.Fixup();
542 fixups_needed.Clear();
543 }
544
545 internal void PushFixup(IFixup f) {
546 fixups_needed.Add(f);
547 }
548
2ff6525 @sorear Thaw routines for types
authored
549 internal void PushRevalidate(STable f) {
550 revalidate.Add(f);
551 }
552
b25256f @sorear Add Serialize.cs to build, fix build errors
authored
553 public byte Byte() { return data[rpointer++]; }
554
19ada5b @sorear Implement variable-length coding for serialization
authored
555 public long Long() {
556 int shift = 0;
557 long accum = 0;
558 while (true) {
559 byte b = Byte();
560 accum |= (((long)(b & 127)) << shift);
561 shift += 7;
562 if ((b & 128) == 0) {
3a82e24 @sorear Fix deserialization of 64-bit values between -2**62-1 and -2**63
authored
563 if ((b & 64) != 0 && shift < 64) {
19ada5b @sorear Implement variable-length coding for serialization
authored
564 accum |= ((-1L) << shift);
565 }
566 //Console.WriteLine("Read {0} end {1}", accum, rpointer);
567 return accum;
568 }
569 }
b25256f @sorear Add Serialize.cs to build, fix build errors
authored
570 }
571
19ada5b @sorear Implement variable-length coding for serialization
authored
572 public ulong ULong() {
573 int shift = 0;
574 ulong accum = 0;
575 while (true) {
576 byte b = Byte();
577 accum |= (((ulong)(b & 127)) << shift);
578 shift += 7;
579 if ((b & 128) == 0) {
580 //Console.WriteLine("Read {0} end {1}", accum, rpointer);
581 return accum;
582 }
583 }
b25256f @sorear Add Serialize.cs to build, fix build errors
authored
584 }
585
19ada5b @sorear Implement variable-length coding for serialization
authored
586 public short Short() {
587 return checked((short)Long());
588 }
589
590 public int Int() {
591 return checked((int)Long());
b25256f @sorear Add Serialize.cs to build, fix build errors
authored
592 }
593
8035d26 @sorear Thaw code for VarDeque, VarHash, BigInteger, Complex, Rat, FatRat
authored
594 public double Double() {
595 return BitConverter.Int64BitsToDouble(Long());
596 }
597
b25256f @sorear Add Serialize.cs to build, fix build errors
authored
598 public string String() {
599 int l = Int();
600
601 if (l < 0) return null;
602 char[] cb = new char[l];
603
604 for (int i = 0; i < l; i++)
19ada5b @sorear Implement variable-length coding for serialization
authored
605 cb[i] = (char)ULong();
b25256f @sorear Add Serialize.cs to build, fix build errors
authored
606
607 return new string(cb);
608 }
609
610 public byte[] Bytes(int k) {
611 byte[] buf = new byte[k];
612
613 for (int i = 0; i < k; i++)
614 buf[i] = Byte();
615
616 return buf;
617 }
618
1c5bb04 @sorear Small refactor to AssemblyBuilder setup, start thaw coding
authored
619 public List<T> RefsL<T>() where T : class {
620 int ct = Int();
621 if (ct < 0) return null;
622 List<T> ret = new List<T>();
623 for (int i = 0; i < ct; i++)
624 ret.Add((T) ObjRef());
625 return ret;
626 }
627
2ff6525 @sorear Thaw routines for types
authored
628 public int[] Ints() {
629 int ct = Int();
630 if (ct < 0) return null;
631 int[] ret = new int[ct];
632 for (int i = 0; i < ct; i++) ret[i] = Int();
633 return ret;
634 }
635
636 public string[] Strings() {
637 int ct = Int();
638 if (ct < 0) return null;
639 string[] ret = new string[ct];
640 for (int i = 0; i < ct; i++) ret[i] = String();
641 return ret;
642 }
643
1c5bb04 @sorear Small refactor to AssemblyBuilder setup, start thaw coding
authored
644 public T[] RefsA<T>() where T : class {
645 int ct = Int();
646 if (ct < 0) return null;
647 T[] ret = new T[ct];
648 for (int i = 0; i < ct; i++)
649 ret[i] = (T) ObjRef();
650 return ret;
651 }
652
604037a @sorear Fix generic thawing of array objects
authored
653 // used from ObjRef only so guaranteed non-null
654 T[] RefsARegister<T>() where T : class {
655 int ct = Int();
656 T[] ret = new T[ct];
657 Register(ret);
658 for (int i = 0; i < ct; i++)
659 ret[i] = (T) ObjRef();
660 return ret;
661 }
662
b25256f @sorear Add Serialize.cs to build, fix build errors
authored
663 public object ObjRef() {
8184302 @sorear Start draft of serialization/deserialization code
authored
664 var tag = (SerializationCode)Byte();
19ada5b @sorear Implement variable-length coding for serialization
authored
665 if (Config.SerTrace)
666 Console.WriteLine("Reading {0} from {1}...", tag, rpointer-1);
8184302 @sorear Start draft of serialization/deserialization code
authored
667 int i, j;
668 switch(tag) {
d832c02 @sorear Second draft of serialization code
authored
669 case SerializationCode.Null:
670 return null;
8035d26 @sorear Thaw code for VarDeque, VarHash, BigInteger, Complex, Rat, FatRat
authored
671
8184302 @sorear Start draft of serialization/deserialization code
authored
672 case SerializationCode.ForeignRef:
673 i = Int();
674 j = Int();
d832c02 @sorear Second draft of serialization code
authored
675 return unit_map[i].bynum[j];
8035d26 @sorear Thaw code for VarDeque, VarHash, BigInteger, Complex, Rat, FatRat
authored
676 case SerializationCode.SelfRef:
677 i = Int();
678 return unit.bynum[i];
8184302 @sorear Start draft of serialization/deserialization code
authored
679 case SerializationCode.NewUnitRef:
d832c02 @sorear Second draft of serialization code
authored
680 return LoadNewUnit();
8035d26 @sorear Thaw code for VarDeque, VarHash, BigInteger, Complex, Rat, FatRat
authored
681
1c5bb04 @sorear Small refactor to AssemblyBuilder setup, start thaw coding
authored
682 case SerializationCode.RuntimeUnit:
683 return RuntimeUnit.Thaw(this);
2671ab4 @sorear Add thaw routine for SubInfo
authored
684 case SerializationCode.SubInfo:
685 return SubInfo.Thaw(this);
2ff6525 @sorear Thaw routines for types
authored
686 case SerializationCode.STable:
687 return STable.Thaw(this);
fe6e595 @sorear Thaw for SimpleVariable, StashEnt
authored
688 case SerializationCode.StashEnt:
689 return StashEnt.Thaw(this);
8035d26 @sorear Thaw code for VarDeque, VarHash, BigInteger, Complex, Rat, FatRat
authored
690 case SerializationCode.Rat:
691 return Rat.Thaw(this);
692 case SerializationCode.FatRat:
693 return FatRat.Thaw(this);
694 case SerializationCode.Complex:
695 return Complex.Thaw(this);
696 case SerializationCode.BigInteger:
697 return BigInteger.Thaw(this);
698 case SerializationCode.VarDeque:
699 return VarDeque.Thaw(this);
700 case SerializationCode.VarHash:
701 return VarHash.Thaw(this);
be108f9 @sorear Thaw routines for P6opaque, DispatchEnt, double, int, Type, string, bool...
authored
702 case SerializationCode.DispatchEnt:
703 return DispatchEnt.Thaw(this);
8035d26 @sorear Thaw code for VarDeque, VarHash, BigInteger, Complex, Rat, FatRat
authored
704 //case SerializationCode.RxFrame:
705 // return RxFrame.Thaw(this);
2ff6525 @sorear Thaw routines for types
authored
706 case SerializationCode.P6how:
707 return P6how.Thaw(this);
7ae298f @sorear Make CC, LIHint serializable
authored
708 case SerializationCode.CC:
709 return CC.Thaw(this);
e7d3f3f @sorear Reimplement regex_infix:<|> table generation
authored
710 case SerializationCode.AltInfo:
711 return AltInfo.Thaw(this);
10ba66d @sorear Turn Signatures into real objects
authored
712 case SerializationCode.Signature:
713 return Signature.Thaw(this);
714 case SerializationCode.Parameter:
715 return Parameter.Thaw(this);
8035d26 @sorear Thaw code for VarDeque, VarHash, BigInteger, Complex, Rat, FatRat
authored
716
3f80fca @sorear A procedure for serializing ContextHandler et al
authored
717 case SerializationCode.ReflectObj:
718 return ReflectObj.Thaw(this);
be108f9 @sorear Thaw routines for P6opaque, DispatchEnt, double, int, Type, string, bool...
authored
719 case SerializationCode.P6opaque:
720 return P6opaque.Thaw(this);
9b6504a @sorear Thaw routines for Frame
authored
721 case SerializationCode.Frame:
722 return Frame.Thaw(this);
8035d26 @sorear Thaw code for VarDeque, VarHash, BigInteger, Complex, Rat, FatRat
authored
723 //Cursor,
724
be108f9 @sorear Thaw routines for P6opaque, DispatchEnt, double, int, Type, string, bool...
authored
725 case SerializationCode.String:
726 return Register(String());
727 case SerializationCode.ArrP6any:
604037a @sorear Fix generic thawing of array objects
authored
728 return RefsARegister<P6any>();
be108f9 @sorear Thaw routines for P6opaque, DispatchEnt, double, int, Type, string, bool...
authored
729 case SerializationCode.ArrVariable:
604037a @sorear Fix generic thawing of array objects
authored
730 return RefsARegister<Variable>();
7ae298f @sorear Make CC, LIHint serializable
authored
731 case SerializationCode.ArrString:
604037a @sorear Fix generic thawing of array objects
authored
732 return RefsARegister<string>();
7ae298f @sorear Make CC, LIHint serializable
authored
733 case SerializationCode.ArrCC:
604037a @sorear Fix generic thawing of array objects
authored
734 return RefsARegister<CC>();
be108f9 @sorear Thaw routines for P6opaque, DispatchEnt, double, int, Type, string, bool...
authored
735 case SerializationCode.Boolean:
736 return Register(Byte() != 0);
737 case SerializationCode.Int:
738 return Register(Int());
739 case SerializationCode.Double:
740 return Register(Double());
741 case SerializationCode.Type:
742 return Register(Type.GetType(String(), true));
8035d26 @sorear Thaw code for VarDeque, VarHash, BigInteger, Complex, Rat, FatRat
authored
743
fe6e595 @sorear Thaw for SimpleVariable, StashEnt
authored
744 case SerializationCode.SimpleVariable:
745 case SerializationCode.SimpleVariable_1:
746 case SerializationCode.SimpleVariable_2:
747 case SerializationCode.SimpleVariable_3:
748 return SimpleVariable.Thaw(this,
749 (int)tag - (int)SerializationCode.SimpleVariable);
054561b @sorear Thaw code for all LAD, Variable, ViviHook subclasses
authored
750 case SerializationCode.SubstrLValue:
751 return SubstrLValue.Thaw(this);
752 case SerializationCode.TiedVariable:
753 return TiedVariable.Thaw(this);
037f4b7 @sorear Make * a "blackhole" container that ignores writes
authored
754 case SerializationCode.Blackhole:
755 return Builtins.Blackhole.Thaw(this);
054561b @sorear Thaw code for all LAD, Variable, ViviHook subclasses
authored
756
757 case SerializationCode.SubViviHook:
758 return SubViviHook.Thaw(this);
759 case SerializationCode.ArrayViviHook:
760 return ArrayViviHook.Thaw(this);
761 case SerializationCode.NewArrayViviHook:
762 return NewArrayViviHook.Thaw(this);
763 case SerializationCode.HashViviHook:
764 return HashViviHook.Thaw(this);
765 case SerializationCode.NewHashViviHook:
766 return NewHashViviHook.Thaw(this);
767
768 case SerializationCode.LADNone:
769 return Register(new LADNone());
770 case SerializationCode.LADNull:
771 return Register(new LADNull());
772 case SerializationCode.LADDot:
773 return Register(new LADDot());
774 case SerializationCode.LADDispatcher:
775 return Register(new LADDispatcher());
776 case SerializationCode.LADImp:
777 return Register(new LADImp());
778 case SerializationCode.LADStr:
779 return LADStr.Thaw(this);
780 case SerializationCode.LADStrNoCase:
781 return LADStrNoCase.Thaw(this);
782 case SerializationCode.LADMethod:
783 return LADMethod.Thaw(this);
784 case SerializationCode.LADParam:
785 return LADParam.Thaw(this);
b8a5ff0 @sorear Treat quantifiers as partially declarative whenever at all possible
authored
786 case SerializationCode.LADQuant:
787 return LADQuant.Thaw(this);
054561b @sorear Thaw code for all LAD, Variable, ViviHook subclasses
authored
788 case SerializationCode.LADSequence:
789 return LADSequence.Thaw(this);
790 case SerializationCode.LADAny:
791 return LADAny.Thaw(this);
792 case SerializationCode.LADCC:
793 return LADCC.Thaw(this);
8184302 @sorear Start draft of serialization/deserialization code
authored
794 default:
1c5bb04 @sorear Small refactor to AssemblyBuilder setup, start thaw coding
authored
795 throw new ThawException("unexpected object tag " + tag);
8184302 @sorear Start draft of serialization/deserialization code
authored
796 }
797 }
d832c02 @sorear Second draft of serialization code
authored
798
1c5bb04 @sorear Small refactor to AssemblyBuilder setup, start thaw coding
authored
799 // call this when thawing any new object
be108f9 @sorear Thaw routines for P6opaque, DispatchEnt, double, int, Type, string, bool...
authored
800 internal object Register(object o) {
1c5bb04 @sorear Small refactor to AssemblyBuilder setup, start thaw coding
authored
801 reg.RegisterThawed(unit, o);
be108f9 @sorear Thaw routines for P6opaque, DispatchEnt, double, int, Type, string, bool...
authored
802 return o;
1c5bb04 @sorear Small refactor to AssemblyBuilder setup, start thaw coding
authored
803 }
804
d832c02 @sorear Second draft of serialization code
authored
805 object LoadNewUnit() {
806 string name = String();
807 if (refed_units == unit_map.Length)
808 Array.Resize(ref unit_map, refed_units * 2);
809
810 SerUnit su = reg.LoadUnit(name);
811 unit_map[refed_units++] = su;
812
813 byte[] hash = Bytes(su.hash.Length);
814
815 for (int i = 0; i < hash.Length; i++)
816 if (hash[i] != su.hash[i])
817 goto badhash;
818
819 int ix = Int();
820 return su.bynum[ix];
821
822 badhash:
eee0cbb @sorear Add framework for detecting stale precompiled modules
authored
823 throw new ThawException(string.Format("Hash mismatch for " +
824 "unit {0} referenced from {1}, wanted {2}, got {3}",
825 su.name, unit.name, Utils.HashToStr(unit.hash),
826 Utils.HashToStr(su.hash)));
d832c02 @sorear Second draft of serialization code
authored
827 }
828 }
829
830 // Thrown to indicate data format problems in the serialized stream
831 // Not necessarily bugs; could also indicate stale files, including
832 // cases where the data format is changed and cases where a depended
833 // file was recreated
834 class ThawException : Exception {
835 public ThawException(string s) : base(s) { }
836 public ThawException() : base() { }
8184302 @sorear Start draft of serialization/deserialization code
authored
837 }
3f80fca @sorear A procedure for serializing ContextHandler et al
authored
838
839 public class ReflectObj : IFreeze {
840 protected virtual object[] GetData() { return new object[0]; }
841 protected virtual void SetData(object[] a) { }
842 void IFreeze.Freeze(FreezeBuffer fb) {
843 fb.Byte((byte)SerializationCode.ReflectObj);
844 fb.String(GetType().AssemblyQualifiedName);
845 fb.Refs(GetData());
846 }
847
848 internal static ReflectObj Thaw(ThawBuffer tb) {
ead51d9 @sorear Skeleton of -gen-app; use named methods for builtin methods to reduce fr...
authored
849 string nm = tb.String();
850 if (Backend.cross_level_load)
851 nm = nm.Replace("Run.", "");
852 Type nt = Type.GetType(nm, true);
3f80fca @sorear A procedure for serializing ContextHandler et al
authored
853 ReflectObj n = (ReflectObj)
f0788e8 @sorear Small bugfixes, use correct form of Invoke for .net
authored
854 nt.GetConstructor(new Type[0]).Invoke(new object[0]);
3f80fca @sorear A procedure for serializing ContextHandler et al
authored
855 tb.Register(n);
856 n.SetData(tb.RefsA<object>());
857 return n;
858 }
859 }
8184302 @sorear Start draft of serialization/deserialization code
authored
860 }
Something went wrong with that request. Please try again.