Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Newer
Older
100644 899 lines (780 sloc) 29.738 kB
8184302 @sorear Start draft of serialization/deserialization code
authored
1 using System;
d832c02 @sorear Second draft of serialization code
authored
2 using System.IO;
3 using System.Security.Cryptography;
b25256f @sorear Add Serialize.cs to build, fix build errors
authored
4 using System.Collections.Generic;
ee02ddf @sorear Tweaks to allow saving and thawing cross-unit deep sub clones, such a…
authored
5 using System.Reflection;
d832c02 @sorear Second draft of serialization code
authored
6 using System.Text;
9083142 @sorear Switch to single-AppDomain execution
authored
7 using Niecza.CLRBackend;
8184302 @sorear Start draft of serialization/deserialization code
authored
8
cdde7e0 @sorear Notes on four kinds of module scope
authored
9 // Here in Niecza we have four different kinds of unit scopes:
10 //
11 // * COMPILING::UNIT, aka RuntimeUnit: one of these exists for every
12 // call into the compiler, whether eval or module. The REPL will
13 // be considered as if it were eval.
14 //
15 // * Serialization scopes, which are created when compiling modules
16 // or when pre-compiling a main program. During execution there is
17 // no current serialization scope; evals inherit the serialization
18 // scope or lack thereof that was in effect.
19 //
20 // * Assembly scopes, which are an artifact of the CLR and almost align
21 // with precompilation scopes, except that they have to exist always
22 // because methods cannot be created free-floating in CLR 2.x.
23 //
24 // An assembly scope is created for all serialization scopes, and
25 // non-saved anonymous assemblies are created for eval-and-run of
26 // a file and non-BEGIN-time evals.
27 //
28 // * GLOBAL scope is very much like serialization scope except that there
29 // is a "true globals" scope that is used when not serializing.
30
8184302 @sorear Start draft of serialization/deserialization code
authored
31 // This implements "bounded serialization" for Niecza. Unfortunately
32 // the CLR's builtin serialization can't efficiently be made bounded,
33 // and anyway it would be nice if the serialization format could be
34 // transported across backends.
35
36 // TODO: implement a more Storable-like interface.
d832c02 @sorear Second draft of serialization code
authored
37
38 // Note, the serialization subsystem is *NOT* thread safe !
8184302 @sorear Start draft of serialization/deserialization code
authored
39 namespace Niecza.Serialization {
d832c02 @sorear Second draft of serialization code
authored
40 // Information kept on a serialization unit after loading or storing,
41 // but not before storing.
42 class SerUnit {
43 internal string name; // eg "File.Copy"
44 internal byte[] hash; // hash of entire file, filled at write time
8ed36ea @sorear Wire up serializer to run after pre-compiler
authored
45 internal object[] bynum = new object[8]; // objects in unit
d832c02 @sorear Second draft of serialization code
authored
46 internal object root; // the RuntimeUnit object
8ed36ea @sorear Wire up serializer to run after pre-compiler
authored
47 internal int nobj; // = 0
69f2cee @sorear A scheme for serializing CLR wrappers (fixes #117)
authored
48
49 internal bool fake; // true for CLR reference units
d832c02 @sorear Second draft of serialization code
authored
50 }
8184302 @sorear Start draft of serialization/deserialization code
authored
51
52 // The central feature of *bounded* serialization is that object
53 // registries are kept distinct from the (de)serializer, and can
54 // be shared between serialization runs.
55 class ObjectRegistry {
d832c02 @sorear Second draft of serialization code
authored
56 // TODO: investigate a more specialized representation,
57 // ideally not having to hash as many objects
58 struct ObjRef {
b25256f @sorear Add Serialize.cs to build, fix build errors
authored
59 public SerUnit unit;
60 public int id;
8184302 @sorear Start draft of serialization/deserialization code
authored
61 }
d832c02 @sorear Second draft of serialization code
authored
62 Dictionary<object,ObjRef> byref = new Dictionary<object,ObjRef>();
8184302 @sorear Start draft of serialization/deserialization code
authored
63
d832c02 @sorear Second draft of serialization code
authored
64 Dictionary<string,SerUnit> units =
65 new Dictionary<string,SerUnit>();
66
cdb4548 @sorear Checkpoint of global-variable inventory
authored
67 internal static HashAlgorithm NewHash() { return new SHA256Managed(); }
68
ee02ddf @sorear Tweaks to allow saving and thawing cross-unit deep sub clones, such a…
authored
69 internal Dictionary<string,Dictionary<string,MethodInfo>> methods =
70 new Dictionary<string,Dictionary<string,MethodInfo>>();
c592587 @sorear Change constant table to use instance fields
authored
71 internal Dictionary<string,object> instances =
72 new Dictionary<string,object>();
ee02ddf @sorear Tweaks to allow saving and thawing cross-unit deep sub clones, such a…
authored
73
d832c02 @sorear Second draft of serialization code
authored
74 static readonly string signature = "Niecza-Serialized-Module";
c592587 @sorear Change constant table to use instance fields
authored
75 static readonly int version = 30;
d832c02 @sorear Second draft of serialization code
authored
76
77 // Routines for use by serialization code
78 public bool CheckWriteObject(SerUnit into, object o,
79 out SerUnit lui, out int id) {
80 ObjRef or;
81 if (byref.TryGetValue(o, out or)) {
82 lui = or.unit;
83 id = or.id;
8184302 @sorear Start draft of serialization/deserialization code
authored
84 return true;
d832c02 @sorear Second draft of serialization code
authored
85 }
8184302 @sorear Start draft of serialization/deserialization code
authored
86
d832c02 @sorear Second draft of serialization code
authored
87 if (into.nobj == into.bynum.Length)
88 Array.Resize(ref into.bynum, into.nobj * 2);
8184302 @sorear Start draft of serialization/deserialization code
authored
89
b25256f @sorear Add Serialize.cs to build, fix build errors
authored
90 or.unit = lui = into;
d832c02 @sorear Second draft of serialization code
authored
91 id = or.id = into.nobj++;
92 into.bynum[id] = o;
93
94 byref[o] = or;
8184302 @sorear Start draft of serialization/deserialization code
authored
95
96 return false;
97 }
d832c02 @sorear Second draft of serialization code
authored
98
54921c8 @sorear Prevent choking when trying to serialize a multi-dispatch cache
authored
99 // Undo previous method...
100 public void DeleteLastObject(SerUnit into) {
101 byref.Remove(into.bynum[--into.nobj]);
102 into.bynum[into.nobj] = null;
103 }
104
1c5bb04 @sorear Small refactor to AssemblyBuilder setup, start thaw coding
authored
105 public void RegisterThawed(SerUnit into, object o) {
106 ObjRef or;
107 if (into.nobj == into.bynum.Length)
108 Array.Resize(ref into.bynum, into.nobj * 2);
109
110 or.unit = into;
111 or.id = into.nobj++;
112 into.bynum[or.id] = o;
113
114 byref[o] = or;
115 }
116
d832c02 @sorear Second draft of serialization code
authored
117 // Routines for use by compilation manager
69f2cee @sorear A scheme for serializing CLR wrappers (fixes #117)
authored
118 public void InstallFakeUnit(string name, params object[] items) {
119 SerUnit su = new SerUnit();
120 su.fake = true;
121 su.name = name;
122 su.bynum = items;
123 su.hash = new byte[0];
124 units[name] = su;
125
126 ObjRef or;
127 or.unit = su;
128 for (or.id = 0; or.id < items.Length; or.id++)
129 byref[items[or.id]] = or;
130 }
d832c02 @sorear Second draft of serialization code
authored
131
132 // Loads a single unit from the compiled-data directory.
133 // Will throw a ThawException if a stale reference is encountered
134 // or other data format error.
69f2cee @sorear A scheme for serializing CLR wrappers (fixes #117)
authored
135 public SerUnit LoadUnit(string name, bool fake = false) {
d832c02 @sorear Second draft of serialization code
authored
136 SerUnit su;
137
138 // is the unit already loaded?
139 if (units.TryGetValue(name, out su))
140 return su;
141
69f2cee @sorear A scheme for serializing CLR wrappers (fixes #117)
authored
142 if (fake) {
143 if (name.StartsWith("CLR,"))
144 CLRWrapperProvider.LoadWrapper(name.Substring(4));
145 else
146 throw new ThawException("No handler for fake unit name " + name);
147
148 return units[name];
149 }
150
9083142 @sorear Switch to single-AppDomain execution
authored
151 string file = Path.Combine(Backend.obj_dir, Backend.prefix +
152 name.Replace("::",".") + ".ser");
d832c02 @sorear Second draft of serialization code
authored
153 byte[] bytes = File.ReadAllBytes(file);
154
155 su = new SerUnit();
156 su.name = name;
cdb4548 @sorear Checkpoint of global-variable inventory
authored
157 su.hash = NewHash().ComputeHash(bytes);
d832c02 @sorear Second draft of serialization code
authored
158
b344401 @sorear ThawBuffer knows its setting
authored
159 ThawBuffer tb = new ThawBuffer(Compartment.Top, this, su, bytes);
d832c02 @sorear Second draft of serialization code
authored
160
8ed36ea @sorear Wire up serializer to run after pre-compiler
authored
161 units[name] = su;
d832c02 @sorear Second draft of serialization code
authored
162 bool success = false;
163 try {
164 string rsig = tb.String();
165 if (rsig != signature)
166 throw new ThawException("signature mismatch loading " + file);
167 int rver = tb.Int();
168 if (rver != version)
169 throw new ThawException("version mismatch loading " + file);
170
b25256f @sorear Add Serialize.cs to build, fix build errors
authored
171 su.root = tb.ObjRef();
c0af787 @sorear Fix fixup call order (oops!), serialization-compatible calling of pha…
authored
172 tb.RunFixups();
d832c02 @sorear Second draft of serialization code
authored
173 success = true;
174 } finally {
175 // don't leave half-read units in the map
176 if (!success)
177 UnloadUnit(name);
178 }
179
180 return su;
181 }
182
183 // removes a stale unit so a new version can be saved over it.
184 public void UnloadUnit(string name) {
8ed36ea @sorear Wire up serializer to run after pre-compiler
authored
185 if (!units.ContainsKey(name))
186 return;
d832c02 @sorear Second draft of serialization code
authored
187 SerUnit su = units[name];
188 units.Remove(name);
189
190 for (int i = 0; i < su.nobj; i++)
191 byref.Remove(su.bynum[i]);
192 }
193
194 public SerUnit SaveUnit(string name, IFreeze root) {
195 SerUnit su = new SerUnit();
196 su.name = name;
197 su.root = root;
198
199 if (units.ContainsKey(name))
b25256f @sorear Add Serialize.cs to build, fix build errors
authored
200 throw new InvalidOperationException("unit " +name+ " exists");
d832c02 @sorear Second draft of serialization code
authored
201
202 bool success = false;
9083142 @sorear Switch to single-AppDomain execution
authored
203 string file = Path.Combine(Backend.obj_dir, Backend.prefix +
204 name.Replace("::",".") + ".ser");
d832c02 @sorear Second draft of serialization code
authored
205
206 FreezeBuffer fb = new FreezeBuffer(this, su);
207
8ed36ea @sorear Wire up serializer to run after pre-compiler
authored
208 units[name] = su;
209
d832c02 @sorear Second draft of serialization code
authored
210 try {
211 fb.String(signature);
212 fb.Int(version);
213 fb.ObjRef(root);
214
215 byte[] data = fb.GetData();
cdb4548 @sorear Checkpoint of global-variable inventory
authored
216 su.hash = NewHash().ComputeHash(data);
d832c02 @sorear Second draft of serialization code
authored
217 File.WriteAllBytes(file, data);
218 success = true;
219 } finally {
220 if (!success)
221 UnloadUnit(name);
222 }
223
224 return su;
225 }
8184302 @sorear Start draft of serialization/deserialization code
authored
226 }
227
228 // One of these codes is written at the beginning of every object ref
1993029 @sorear Start implementing serialization for our objects
authored
229 enum SerializationCode : byte {
d832c02 @sorear Second draft of serialization code
authored
230 // special
231 Null,
232
8184302 @sorear Start draft of serialization/deserialization code
authored
233 // existing objects
234 ForeignRef,
235 SelfRef,
236 NewUnitRef,
69f2cee @sorear A scheme for serializing CLR wrappers (fixes #117)
authored
237 FakeUnitRef,
1993029 @sorear Start implementing serialization for our objects
authored
238
239 // types of new object
240 RuntimeUnit,
241 SubInfo,
242 STable,
243 StashEnt,
d91ecdf @sorear Serialization for objects of type Rat, FatRat, Complex, BigInteger, I…
authored
244 Rat,
245 FatRat,
246 Complex,
247 BigInteger,
248 VarDeque,
249 VarHash,
dc6dd55 @sorear Implement serialization for Frame, finally kill off hashtable lexicals
authored
250 DispatchEnt,
251 RxFrame,
252 P6how,
3f80fca @sorear A procedure for serializing ContextHandler et al
authored
253 ReflectObj,
7ae298f @sorear Make CC, LIHint serializable
authored
254 CC,
255 AltInfo,
10ba66d @sorear Turn Signatures into real objects
authored
256 Signature,
257 Parameter,
1993029 @sorear Start implementing serialization for our objects
authored
258
259 // types of P6any-reified object
d91ecdf @sorear Serialization for objects of type Rat, FatRat, Complex, BigInteger, I…
authored
260 P6opaque, // eventually let's specialize this
1993029 @sorear Start implementing serialization for our objects
authored
261 Frame,
262 Cursor,
263
1c5bb04 @sorear Small refactor to AssemblyBuilder setup, start thaw coding
authored
264 // miscellany - keep these in same order as FallbackFreeze
265 String,
266 ArrP6any,
267 ArrVariable,
7ae298f @sorear Make CC, LIHint serializable
authored
268 ArrString,
269 ArrCC,
fe6e595 @sorear Thaw for SimpleVariable, StashEnt
authored
270 Boolean,
1c5bb04 @sorear Small refactor to AssemblyBuilder setup, start thaw coding
authored
271 Int,
272 Double,
3f80fca @sorear A procedure for serializing ContextHandler et al
authored
273 Type,
d91ecdf @sorear Serialization for objects of type Rat, FatRat, Complex, BigInteger, I…
authored
274
ab35238 @sorear Tidy up STable fields a bit, add TiedVariable serialize
authored
275 // variables
40e1779 @sorear Split ListVariable out of SimpleVariable
authored
276 RWVariable, // allow 2 for flag
277 RWVariable_1,
278 ListVariable,
fe6e595 @sorear Thaw for SimpleVariable, StashEnt
authored
279 SubstrLValue,
ab35238 @sorear Tidy up STable fields a bit, add TiedVariable serialize
authored
280 TiedVariable,
037f4b7 @sorear Make * a "blackhole" container that ignores writes
authored
281 Blackhole,
1993029 @sorear Start implementing serialization for our objects
authored
282
283 // vivification hooks
ab35238 @sorear Tidy up STable fields a bit, add TiedVariable serialize
authored
284 SubViviHook,
1993029 @sorear Start implementing serialization for our objects
authored
285 ArrayViviHook,
286 NewArrayViviHook,
287 HashViviHook,
288 NewHashViviHook,
b1088dc @sorear implement serialization for SubInfo, LAD, LexInfo
authored
289
290 // Longest-token automaton descriptors
291 LADNone, // no-args
292 LADNull,
293 LADDot,
294 LADDispatcher,
295 LADImp,
296 LADStr, // string
297 LADStrNoCase,
298 LADMethod,
299 LADParam,
b8a5ff0 @sorear Treat quantifiers as partially declarative whenever at all possible
authored
300 LADQuant, // byte,LAD,LAD
b1088dc @sorear implement serialization for SubInfo, LAD, LexInfo
authored
301 LADSequence, // LAD[]
302 LADAny,
303 LADCC, // CC
8184302 @sorear Start draft of serialization/deserialization code
authored
304 }
305
306 // An instance of this class is used to serialize serialization units
1993029 @sorear Start implementing serialization for our objects
authored
307 public class FreezeBuffer {
8184302 @sorear Start draft of serialization/deserialization code
authored
308 byte[] data;
309 int wpointer;
310
d832c02 @sorear Second draft of serialization code
authored
311 Dictionary<SerUnit,int> unit_to_offset;
312 int usedunits;
8184302 @sorear Start draft of serialization/deserialization code
authored
313
314 ObjectRegistry reg;
d832c02 @sorear Second draft of serialization code
authored
315 SerUnit unit;
8184302 @sorear Start draft of serialization/deserialization code
authored
316
d832c02 @sorear Second draft of serialization code
authored
317 internal FreezeBuffer(ObjectRegistry reg, SerUnit unit) {
8ed36ea @sorear Wire up serializer to run after pre-compiler
authored
318 if (reg == null || unit == null)
319 throw new ArgumentNullException();
8184302 @sorear Start draft of serialization/deserialization code
authored
320 this.reg = reg;
d832c02 @sorear Second draft of serialization code
authored
321 this.unit = unit;
322 unit_to_offset = new Dictionary<SerUnit,int>();
8184302 @sorear Start draft of serialization/deserialization code
authored
323 data = new byte[256];
324 }
325
b25256f @sorear Add Serialize.cs to build, fix build errors
authored
326 internal byte[] GetData() {
327 byte[] ret = new byte[wpointer];
328 Array.Copy(data, ret, ret.Length);
329 return ret;
330 }
331
8184302 @sorear Start draft of serialization/deserialization code
authored
332 void Ensure(int ct) {
333 while (ct + wpointer > data.Length)
334 Array.Resize(ref data, data.Length * 2);
335 }
336
337 public void Byte(byte x) {
338 Ensure(1);
339 data[wpointer++] = x;
340 }
341
19ada5b @sorear Implement variable-length coding for serialization
authored
342 public void Long(long x) {
343 //Console.WriteLine("Saving {0} at {1}", x, wpointer);
344 Ensure(10);
345 while (true) {
346 if (x >= -64 && x <= 63) {
347 data[wpointer++] = (byte) (127 & (byte)x);
348 break;
349 } else {
350 data[wpointer++] = (byte) (128 | (byte)x);
351 x >>= 7;
352 }
353 }
8184302 @sorear Start draft of serialization/deserialization code
authored
354 }
355
19ada5b @sorear Implement variable-length coding for serialization
authored
356 public void ULong(ulong x) {
357 //Console.WriteLine("Saving {0} at {1}", x, wpointer);
358 Ensure(10);
359 while (true) {
360 if (x <= 127) {
361 data[wpointer++] = (byte) (127 & (byte)x);
362 break;
363 } else {
364 data[wpointer++] = (byte) (128 | (byte)x);
365 x >>= 7;
366 }
367 }
8184302 @sorear Start draft of serialization/deserialization code
authored
368 }
369
19ada5b @sorear Implement variable-length coding for serialization
authored
370 public void Short(short x) { Long(x); }
371 public void Int(int x) { Long(x); }
8184302 @sorear Start draft of serialization/deserialization code
authored
372
8035d26 @sorear Thaw code for VarDeque, VarHash, BigInteger, Complex, Rat, FatRat
authored
373 public void Double(double x) {
374 Long(BitConverter.DoubleToInt64Bits(x));
375 }
376
8184302 @sorear Start draft of serialization/deserialization code
authored
377 public void String(string s) {
378 if (s == null) {
379 Int(-1);
380 } else {
381 Int(s.Length);
382 foreach (char ch in s)
19ada5b @sorear Implement variable-length coding for serialization
authored
383 ULong((ulong)ch);
8184302 @sorear Start draft of serialization/deserialization code
authored
384 }
385 }
386
3dc4789 @sorear Implement freezing for types
authored
387 public void Strings(string[] s) {
388 if (s == null) Int(-1);
389 else {
390 Int(s.Length);
391 foreach (string ch in s) String(ch);
392 }
393 }
394
b1088dc @sorear implement serialization for SubInfo, LAD, LexInfo
authored
395 public void Ints(int[] s) {
396 if (s == null) {
397 Int(-1);
398 } else {
399 Int(s.Length);
400 foreach (int ch in s)
401 Int(ch);
402 }
403 }
404
dc6dd55 @sorear Implement serialization for Frame, finally kill off hashtable lexicals
authored
405 public void Refs<T> (T[] x) {
d91ecdf @sorear Serialization for objects of type Rat, FatRat, Complex, BigInteger, I…
authored
406 if (x == null) {
407 Int(-1);
408 } else {
409 Int(x.Length);
410 foreach (T y in x)
411 ObjRef(y);
412 }
413 }
414
dc6dd55 @sorear Implement serialization for Frame, finally kill off hashtable lexicals
authored
415 public void Refs<T> (IList<T> x) {
3dc4789 @sorear Implement freezing for types
authored
416 if (x == null) {
417 Int(-1);
418 } else {
419 Int(x.Count);
420 foreach (T y in x)
421 ObjRef(y);
422 }
423 }
424
54921c8 @sorear Prevent choking when trying to serialize a multi-dispatch cache
authored
425 public void Ephemeralize() {
426 Byte((byte)SerializationCode.Null);
427 reg.DeleteLastObject(unit);
428 }
429
d832c02 @sorear Second draft of serialization code
authored
430 // This is the main routine you should call from your Freeze
431 // callbacks to freeze an object
ff557f5 @sorear Usability improvements for NIECZA_SER_TRACE, indenting
authored
432 static int depth;
d91ecdf @sorear Serialization for objects of type Rat, FatRat, Complex, BigInteger, I…
authored
433 public void ObjRef(object o) {
d832c02 @sorear Second draft of serialization code
authored
434 int id;
435 SerUnit altunit;
ff557f5 @sorear Usability improvements for NIECZA_SER_TRACE, indenting
authored
436 if (Config.SerTrace) {
437 Console.WriteLine("{0}Saving {1} at {2}:", new string(' ',depth), o, wpointer);
438 }
439
d832c02 @sorear Second draft of serialization code
authored
440 if (o == null) { // null pointers are special
441 Byte((byte)SerializationCode.Null);
442 return;
443 }
444
445 if (reg.CheckWriteObject(unit, o, out altunit, out id)) {
8184302 @sorear Start draft of serialization/deserialization code
authored
446 if (altunit == unit) {
447 Byte((byte)SerializationCode.SelfRef);
448 } else {
d832c02 @sorear Second draft of serialization code
authored
449 int altcode;
450 if (!unit_to_offset.TryGetValue(altunit, out altcode)) {
69f2cee @sorear A scheme for serializing CLR wrappers (fixes #117)
authored
451 Byte(altunit.fake ?
452 (byte)SerializationCode.FakeUnitRef :
453 (byte)SerializationCode.NewUnitRef);
b25256f @sorear Add Serialize.cs to build, fix build errors
authored
454 String(altunit.name);
d832c02 @sorear Second draft of serialization code
authored
455 // save the hash too so stale refs can be caught
456 foreach (byte b in altunit.hash) Byte(b);
457
8184302 @sorear Start draft of serialization/deserialization code
authored
458 unit_to_offset[altunit] = usedunits++;
459 } else {
460 Byte((byte)SerializationCode.ForeignRef);
d832c02 @sorear Second draft of serialization code
authored
461 Int(altcode);
8184302 @sorear Start draft of serialization/deserialization code
authored
462 }
463 }
464 Int((int)id);
465 } else {
d832c02 @sorear Second draft of serialization code
authored
466 // must take responsibility for saving the tag
d91ecdf @sorear Serialization for objects of type Rat, FatRat, Complex, BigInteger, I…
authored
467 IFreeze f = o as IFreeze;
ff557f5 @sorear Usability improvements for NIECZA_SER_TRACE, indenting
authored
468 if (Config.SerTrace) depth++;
d91ecdf @sorear Serialization for objects of type Rat, FatRat, Complex, BigInteger, I…
authored
469 if (f != null) {
470 f.Freeze(this);
471 } else {
472 FallbackFreeze(o);
473 }
ff557f5 @sorear Usability improvements for NIECZA_SER_TRACE, indenting
authored
474 if (Config.SerTrace) depth--;
d91ecdf @sorear Serialization for objects of type Rat, FatRat, Complex, BigInteger, I…
authored
475 }
476 }
477
cdb4548 @sorear Checkpoint of global-variable inventory
authored
478 [Immutable]
be108f9 @sorear Thaw routines for P6opaque, DispatchEnt, double, int, Type, string, b…
authored
479 internal static Type[] boxTypes = new Type[] {
480 null, typeof(Rat), typeof(FatRat), typeof(Complex),
7ae298f @sorear Make CC, LIHint serializable
authored
481 typeof(double), typeof(int), typeof(string), typeof(VarHash),
be108f9 @sorear Thaw routines for P6opaque, DispatchEnt, double, int, Type, string, b…
authored
482 typeof(Variable[]), typeof(VarDeque), typeof(STable),
eba5eb3 @sorear Take a first crack at implementing constant folding
authored
483 typeof(BigInteger),
be108f9 @sorear Thaw routines for P6opaque, DispatchEnt, double, int, Type, string, b…
authored
484 };
cdb4548 @sorear Checkpoint of global-variable inventory
authored
485 [Immutable]
be108f9 @sorear Thaw routines for P6opaque, DispatchEnt, double, int, Type, string, b…
authored
486 internal static Func<P6opaque>[] boxCreate = new Func<P6opaque>[] {
487 P6opaque.Create, BoxObject<Rat>.Create, BoxObject<FatRat>.Create,
488 BoxObject<Complex>.Create, BoxObject<double>.Create,
489 BoxObject<int>.Create, BoxObject<string>.Create,
19ada5b @sorear Implement variable-length coding for serialization
authored
490 BoxObject<VarHash>.Create, BoxObject<Variable[]>.Create,
491 BoxObject<VarDeque>.Create, BoxObject<STable>.Create,
eba5eb3 @sorear Take a first crack at implementing constant folding
authored
492 BoxObject<BigInteger>.Create,
be108f9 @sorear Thaw routines for P6opaque, DispatchEnt, double, int, Type, string, b…
authored
493 };
cdb4548 @sorear Checkpoint of global-variable inventory
authored
494 [Immutable]
d91ecdf @sorear Serialization for objects of type Rat, FatRat, Complex, BigInteger, I…
authored
495 static Type[] anyTypes = new Type[] {
496 typeof(string), typeof(P6any[]), typeof(Variable[]),
7ae298f @sorear Make CC, LIHint serializable
authored
497 typeof(string[]), typeof(CC[]),
3f80fca @sorear A procedure for serializing ContextHandler et al
authored
498 typeof(bool), typeof(int), typeof(double), typeof(Type),
d91ecdf @sorear Serialization for objects of type Rat, FatRat, Complex, BigInteger, I…
authored
499 };
500
501 void FallbackFreeze(object o) {
502 int ix = 0;
503 Type t = o.GetType();
fe6e595 @sorear Thaw for SimpleVariable, StashEnt
authored
504 while (ix != anyTypes.Length && anyTypes[ix] != t) ix++;
1c5bb04 @sorear Small refactor to AssemblyBuilder setup, start thaw coding
authored
505 Byte((byte)(((int)SerializationCode.String) + ix));
d91ecdf @sorear Serialization for objects of type Rat, FatRat, Complex, BigInteger, I…
authored
506
507 switch(ix) {
508 case 0:
509 String((string)o);
510 break;
511 case 1:
512 Refs((P6any[])o);
513 break;
514 case 2:
515 Refs((Variable[])o);
516 break;
517 case 3:
7ae298f @sorear Make CC, LIHint serializable
authored
518 Refs((string[])o);
d91ecdf @sorear Serialization for objects of type Rat, FatRat, Complex, BigInteger, I…
authored
519 break;
520 case 4:
7ae298f @sorear Make CC, LIHint serializable
authored
521 Refs((CC[])o);
fe6e595 @sorear Thaw for SimpleVariable, StashEnt
authored
522 break;
523 case 5:
7ae298f @sorear Make CC, LIHint serializable
authored
524 Byte((byte)((bool)o ? 1 : 0));
d91ecdf @sorear Serialization for objects of type Rat, FatRat, Complex, BigInteger, I…
authored
525 break;
3f80fca @sorear A procedure for serializing ContextHandler et al
authored
526 case 6:
7ae298f @sorear Make CC, LIHint serializable
authored
527 Int((int)o);
528 break;
529 case 7:
530 Double((double)o);
531 break;
532 case 8:
3f80fca @sorear A procedure for serializing ContextHandler et al
authored
533 String(((Type)o).AssemblyQualifiedName);
534 break;
d91ecdf @sorear Serialization for objects of type Rat, FatRat, Complex, BigInteger, I…
authored
535 default:
536 throw new NotImplementedException(t.FullName);
8184302 @sorear Start draft of serialization/deserialization code
authored
537 }
538 }
539 }
540
541 // Note that this interface only handles freezing - thaw is done using
542 // a switch statement.
1993029 @sorear Start implementing serialization for our objects
authored
543 public interface IFreeze {
8184302 @sorear Start draft of serialization/deserialization code
authored
544 void Freeze(FreezeBuffer fb);
545 }
fe6e595 @sorear Thaw for SimpleVariable, StashEnt
authored
546 // implement this if you need to copy in data from other objects, &c
547 interface IFixup {
548 void Fixup();
549 }
8184302 @sorear Start draft of serialization/deserialization code
authored
550
551 class ThawBuffer {
552 byte[] data;
553 int rpointer;
b344401 @sorear ThawBuffer knows its setting
authored
554 internal readonly ObjectRegistry reg;
555 internal readonly Compartment setting;
8184302 @sorear Start draft of serialization/deserialization code
authored
556
d832c02 @sorear Second draft of serialization code
authored
557 SerUnit[] unit_map = new SerUnit[8];
8184302 @sorear Start draft of serialization/deserialization code
authored
558 int refed_units;
d832c02 @sorear Second draft of serialization code
authored
559 SerUnit unit;
8184302 @sorear Start draft of serialization/deserialization code
authored
560
fe6e595 @sorear Thaw for SimpleVariable, StashEnt
authored
561 List<IFixup> fixups_needed = new List<IFixup>();
2ff6525 @sorear Thaw routines for types
authored
562 List<object> revalidate = new List<object>();
fe6e595 @sorear Thaw for SimpleVariable, StashEnt
authored
563
288db48 @sorear Use precompiled IL when using a precompiled module
authored
564 public Type type;
565
b344401 @sorear ThawBuffer knows its setting
authored
566 internal ThawBuffer(Compartment setting, ObjectRegistry reg, SerUnit unit, byte[] data) {
567 this.data = data;
568 this.setting = setting;
569 this.reg = reg;
570 this.unit = unit;
8184302 @sorear Start draft of serialization/deserialization code
authored
571 }
572
fe6e595 @sorear Thaw for SimpleVariable, StashEnt
authored
573 internal void RunFixups() {
2ff6525 @sorear Thaw routines for types
authored
574 P6how.BulkRevalidate(revalidate);
fe6e595 @sorear Thaw for SimpleVariable, StashEnt
authored
575 foreach (IFixup f in fixups_needed)
576 f.Fixup();
577 fixups_needed.Clear();
578 }
579
580 internal void PushFixup(IFixup f) {
581 fixups_needed.Add(f);
582 }
583
2ff6525 @sorear Thaw routines for types
authored
584 internal void PushRevalidate(STable f) {
585 revalidate.Add(f);
586 }
587
b25256f @sorear Add Serialize.cs to build, fix build errors
authored
588 public byte Byte() { return data[rpointer++]; }
589
19ada5b @sorear Implement variable-length coding for serialization
authored
590 public long Long() {
591 int shift = 0;
592 long accum = 0;
593 while (true) {
594 byte b = Byte();
595 accum |= (((long)(b & 127)) << shift);
596 shift += 7;
597 if ((b & 128) == 0) {
3a82e24 @sorear Fix deserialization of 64-bit values between -2**62-1 and -2**63
authored
598 if ((b & 64) != 0 && shift < 64) {
19ada5b @sorear Implement variable-length coding for serialization
authored
599 accum |= ((-1L) << shift);
600 }
601 //Console.WriteLine("Read {0} end {1}", accum, rpointer);
602 return accum;
603 }
604 }
b25256f @sorear Add Serialize.cs to build, fix build errors
authored
605 }
606
19ada5b @sorear Implement variable-length coding for serialization
authored
607 public ulong ULong() {
608 int shift = 0;
609 ulong accum = 0;
610 while (true) {
611 byte b = Byte();
612 accum |= (((ulong)(b & 127)) << shift);
613 shift += 7;
614 if ((b & 128) == 0) {
615 //Console.WriteLine("Read {0} end {1}", accum, rpointer);
616 return accum;
617 }
618 }
b25256f @sorear Add Serialize.cs to build, fix build errors
authored
619 }
620
19ada5b @sorear Implement variable-length coding for serialization
authored
621 public short Short() {
622 return checked((short)Long());
623 }
624
625 public int Int() {
626 return checked((int)Long());
b25256f @sorear Add Serialize.cs to build, fix build errors
authored
627 }
628
8035d26 @sorear Thaw code for VarDeque, VarHash, BigInteger, Complex, Rat, FatRat
authored
629 public double Double() {
630 return BitConverter.Int64BitsToDouble(Long());
631 }
632
b25256f @sorear Add Serialize.cs to build, fix build errors
authored
633 public string String() {
634 int l = Int();
635
636 if (l < 0) return null;
637 char[] cb = new char[l];
638
639 for (int i = 0; i < l; i++)
19ada5b @sorear Implement variable-length coding for serialization
authored
640 cb[i] = (char)ULong();
b25256f @sorear Add Serialize.cs to build, fix build errors
authored
641
642 return new string(cb);
643 }
644
645 public byte[] Bytes(int k) {
646 byte[] buf = new byte[k];
647
648 for (int i = 0; i < k; i++)
649 buf[i] = Byte();
650
651 return buf;
652 }
653
1c5bb04 @sorear Small refactor to AssemblyBuilder setup, start thaw coding
authored
654 public List<T> RefsL<T>() where T : class {
655 int ct = Int();
656 if (ct < 0) return null;
657 List<T> ret = new List<T>();
658 for (int i = 0; i < ct; i++)
659 ret.Add((T) ObjRef());
660 return ret;
661 }
662
2ff6525 @sorear Thaw routines for types
authored
663 public int[] Ints() {
664 int ct = Int();
665 if (ct < 0) return null;
666 int[] ret = new int[ct];
667 for (int i = 0; i < ct; i++) ret[i] = Int();
668 return ret;
669 }
670
671 public string[] Strings() {
672 int ct = Int();
673 if (ct < 0) return null;
674 string[] ret = new string[ct];
675 for (int i = 0; i < ct; i++) ret[i] = String();
676 return ret;
677 }
678
1c5bb04 @sorear Small refactor to AssemblyBuilder setup, start thaw coding
authored
679 public T[] RefsA<T>() where T : class {
680 int ct = Int();
681 if (ct < 0) return null;
682 T[] ret = new T[ct];
683 for (int i = 0; i < ct; i++)
684 ret[i] = (T) ObjRef();
685 return ret;
686 }
687
604037a @sorear Fix generic thawing of array objects
authored
688 // used from ObjRef only so guaranteed non-null
689 T[] RefsARegister<T>() where T : class {
690 int ct = Int();
691 T[] ret = new T[ct];
692 Register(ret);
693 for (int i = 0; i < ct; i++)
694 ret[i] = (T) ObjRef();
695 return ret;
696 }
697
b25256f @sorear Add Serialize.cs to build, fix build errors
authored
698 public object ObjRef() {
8184302 @sorear Start draft of serialization/deserialization code
authored
699 var tag = (SerializationCode)Byte();
19ada5b @sorear Implement variable-length coding for serialization
authored
700 if (Config.SerTrace)
701 Console.WriteLine("Reading {0} from {1}...", tag, rpointer-1);
8184302 @sorear Start draft of serialization/deserialization code
authored
702 int i, j;
703 switch(tag) {
d832c02 @sorear Second draft of serialization code
authored
704 case SerializationCode.Null:
705 return null;
8035d26 @sorear Thaw code for VarDeque, VarHash, BigInteger, Complex, Rat, FatRat
authored
706
8184302 @sorear Start draft of serialization/deserialization code
authored
707 case SerializationCode.ForeignRef:
708 i = Int();
709 j = Int();
d832c02 @sorear Second draft of serialization code
authored
710 return unit_map[i].bynum[j];
8035d26 @sorear Thaw code for VarDeque, VarHash, BigInteger, Complex, Rat, FatRat
authored
711 case SerializationCode.SelfRef:
712 i = Int();
713 return unit.bynum[i];
8184302 @sorear Start draft of serialization/deserialization code
authored
714 case SerializationCode.NewUnitRef:
69f2cee @sorear A scheme for serializing CLR wrappers (fixes #117)
authored
715 return LoadNewUnit(false);
716 case SerializationCode.FakeUnitRef:
717 return LoadNewUnit(true);
8035d26 @sorear Thaw code for VarDeque, VarHash, BigInteger, Complex, Rat, FatRat
authored
718
1c5bb04 @sorear Small refactor to AssemblyBuilder setup, start thaw coding
authored
719 case SerializationCode.RuntimeUnit:
720 return RuntimeUnit.Thaw(this);
2671ab4 @sorear Add thaw routine for SubInfo
authored
721 case SerializationCode.SubInfo:
722 return SubInfo.Thaw(this);
2ff6525 @sorear Thaw routines for types
authored
723 case SerializationCode.STable:
724 return STable.Thaw(this);
fe6e595 @sorear Thaw for SimpleVariable, StashEnt
authored
725 case SerializationCode.StashEnt:
726 return StashEnt.Thaw(this);
8035d26 @sorear Thaw code for VarDeque, VarHash, BigInteger, Complex, Rat, FatRat
authored
727 case SerializationCode.Rat:
728 return Rat.Thaw(this);
729 case SerializationCode.FatRat:
730 return FatRat.Thaw(this);
731 case SerializationCode.Complex:
732 return Complex.Thaw(this);
733 case SerializationCode.BigInteger:
734 return BigInteger.Thaw(this);
735 case SerializationCode.VarDeque:
736 return VarDeque.Thaw(this);
737 case SerializationCode.VarHash:
738 return VarHash.Thaw(this);
be108f9 @sorear Thaw routines for P6opaque, DispatchEnt, double, int, Type, string, b…
authored
739 case SerializationCode.DispatchEnt:
740 return DispatchEnt.Thaw(this);
8035d26 @sorear Thaw code for VarDeque, VarHash, BigInteger, Complex, Rat, FatRat
authored
741 //case SerializationCode.RxFrame:
742 // return RxFrame.Thaw(this);
2ff6525 @sorear Thaw routines for types
authored
743 case SerializationCode.P6how:
744 return P6how.Thaw(this);
7ae298f @sorear Make CC, LIHint serializable
authored
745 case SerializationCode.CC:
746 return CC.Thaw(this);
e7d3f3f @sorear Reimplement regex_infix:<|> table generation
authored
747 case SerializationCode.AltInfo:
748 return AltInfo.Thaw(this);
10ba66d @sorear Turn Signatures into real objects
authored
749 case SerializationCode.Signature:
750 return Signature.Thaw(this);
751 case SerializationCode.Parameter:
752 return Parameter.Thaw(this);
8035d26 @sorear Thaw code for VarDeque, VarHash, BigInteger, Complex, Rat, FatRat
authored
753
3f80fca @sorear A procedure for serializing ContextHandler et al
authored
754 case SerializationCode.ReflectObj:
755 return ReflectObj.Thaw(this);
be108f9 @sorear Thaw routines for P6opaque, DispatchEnt, double, int, Type, string, b…
authored
756 case SerializationCode.P6opaque:
757 return P6opaque.Thaw(this);
9b6504a @sorear Thaw routines for Frame
authored
758 case SerializationCode.Frame:
759 return Frame.Thaw(this);
8035d26 @sorear Thaw code for VarDeque, VarHash, BigInteger, Complex, Rat, FatRat
authored
760 //Cursor,
761
be108f9 @sorear Thaw routines for P6opaque, DispatchEnt, double, int, Type, string, b…
authored
762 case SerializationCode.String:
763 return Register(String());
764 case SerializationCode.ArrP6any:
604037a @sorear Fix generic thawing of array objects
authored
765 return RefsARegister<P6any>();
be108f9 @sorear Thaw routines for P6opaque, DispatchEnt, double, int, Type, string, b…
authored
766 case SerializationCode.ArrVariable:
604037a @sorear Fix generic thawing of array objects
authored
767 return RefsARegister<Variable>();
7ae298f @sorear Make CC, LIHint serializable
authored
768 case SerializationCode.ArrString:
604037a @sorear Fix generic thawing of array objects
authored
769 return RefsARegister<string>();
7ae298f @sorear Make CC, LIHint serializable
authored
770 case SerializationCode.ArrCC:
604037a @sorear Fix generic thawing of array objects
authored
771 return RefsARegister<CC>();
be108f9 @sorear Thaw routines for P6opaque, DispatchEnt, double, int, Type, string, b…
authored
772 case SerializationCode.Boolean:
773 return Register(Byte() != 0);
774 case SerializationCode.Int:
775 return Register(Int());
776 case SerializationCode.Double:
777 return Register(Double());
778 case SerializationCode.Type:
779 return Register(Type.GetType(String(), true));
8035d26 @sorear Thaw code for VarDeque, VarHash, BigInteger, Complex, Rat, FatRat
authored
780
40e1779 @sorear Split ListVariable out of SimpleVariable
authored
781 case SerializationCode.ListVariable:
782 return ListVariable.Thaw(this);
783 case SerializationCode.RWVariable:
784 case SerializationCode.RWVariable_1:
785 return RWVariable.Thaw(this,
786 (int)tag - (int)SerializationCode.RWVariable);
054561b @sorear Thaw code for all LAD, Variable, ViviHook subclasses
authored
787 case SerializationCode.SubstrLValue:
788 return SubstrLValue.Thaw(this);
789 case SerializationCode.TiedVariable:
790 return TiedVariable.Thaw(this);
037f4b7 @sorear Make * a "blackhole" container that ignores writes
authored
791 case SerializationCode.Blackhole:
792 return Builtins.Blackhole.Thaw(this);
054561b @sorear Thaw code for all LAD, Variable, ViviHook subclasses
authored
793
794 case SerializationCode.SubViviHook:
795 return SubViviHook.Thaw(this);
796 case SerializationCode.ArrayViviHook:
797 return ArrayViviHook.Thaw(this);
798 case SerializationCode.NewArrayViviHook:
799 return NewArrayViviHook.Thaw(this);
800 case SerializationCode.HashViviHook:
801 return HashViviHook.Thaw(this);
802 case SerializationCode.NewHashViviHook:
803 return NewHashViviHook.Thaw(this);
804
805 case SerializationCode.LADNone:
806 return Register(new LADNone());
807 case SerializationCode.LADNull:
808 return Register(new LADNull());
809 case SerializationCode.LADDot:
810 return Register(new LADDot());
811 case SerializationCode.LADDispatcher:
812 return Register(new LADDispatcher());
813 case SerializationCode.LADImp:
814 return Register(new LADImp());
815 case SerializationCode.LADStr:
816 return LADStr.Thaw(this);
817 case SerializationCode.LADStrNoCase:
818 return LADStrNoCase.Thaw(this);
819 case SerializationCode.LADMethod:
820 return LADMethod.Thaw(this);
821 case SerializationCode.LADParam:
822 return LADParam.Thaw(this);
b8a5ff0 @sorear Treat quantifiers as partially declarative whenever at all possible
authored
823 case SerializationCode.LADQuant:
824 return LADQuant.Thaw(this);
054561b @sorear Thaw code for all LAD, Variable, ViviHook subclasses
authored
825 case SerializationCode.LADSequence:
826 return LADSequence.Thaw(this);
827 case SerializationCode.LADAny:
828 return LADAny.Thaw(this);
829 case SerializationCode.LADCC:
830 return LADCC.Thaw(this);
8184302 @sorear Start draft of serialization/deserialization code
authored
831 default:
1c5bb04 @sorear Small refactor to AssemblyBuilder setup, start thaw coding
authored
832 throw new ThawException("unexpected object tag " + tag);
8184302 @sorear Start draft of serialization/deserialization code
authored
833 }
834 }
d832c02 @sorear Second draft of serialization code
authored
835
1c5bb04 @sorear Small refactor to AssemblyBuilder setup, start thaw coding
authored
836 // call this when thawing any new object
be108f9 @sorear Thaw routines for P6opaque, DispatchEnt, double, int, Type, string, b…
authored
837 internal object Register(object o) {
1c5bb04 @sorear Small refactor to AssemblyBuilder setup, start thaw coding
authored
838 reg.RegisterThawed(unit, o);
be108f9 @sorear Thaw routines for P6opaque, DispatchEnt, double, int, Type, string, b…
authored
839 return o;
1c5bb04 @sorear Small refactor to AssemblyBuilder setup, start thaw coding
authored
840 }
841
69f2cee @sorear A scheme for serializing CLR wrappers (fixes #117)
authored
842 object LoadNewUnit(bool fake) {
d832c02 @sorear Second draft of serialization code
authored
843 string name = String();
844 if (refed_units == unit_map.Length)
845 Array.Resize(ref unit_map, refed_units * 2);
846
69f2cee @sorear A scheme for serializing CLR wrappers (fixes #117)
authored
847 SerUnit su = reg.LoadUnit(name, fake);
d832c02 @sorear Second draft of serialization code
authored
848 unit_map[refed_units++] = su;
849
850 byte[] hash = Bytes(su.hash.Length);
851
852 for (int i = 0; i < hash.Length; i++)
853 if (hash[i] != su.hash[i])
854 goto badhash;
855
856 int ix = Int();
857 return su.bynum[ix];
858
859 badhash:
eee0cbb @sorear Add framework for detecting stale precompiled modules
authored
860 throw new ThawException(string.Format("Hash mismatch for " +
861 "unit {0} referenced from {1}, wanted {2}, got {3}",
862 su.name, unit.name, Utils.HashToStr(unit.hash),
863 Utils.HashToStr(su.hash)));
d832c02 @sorear Second draft of serialization code
authored
864 }
865 }
866
867 // Thrown to indicate data format problems in the serialized stream
868 // Not necessarily bugs; could also indicate stale files, including
869 // cases where the data format is changed and cases where a depended
870 // file was recreated
871 class ThawException : Exception {
872 public ThawException(string s) : base(s) { }
873 public ThawException() : base() { }
8184302 @sorear Start draft of serialization/deserialization code
authored
874 }
3f80fca @sorear A procedure for serializing ContextHandler et al
authored
875
876 public class ReflectObj : IFreeze {
3d9f4b4 @sorear ContextHandlers know their setting and do not need to use Top
authored
877 internal Compartment setting = Compartment.Top;
3f80fca @sorear A procedure for serializing ContextHandler et al
authored
878 protected virtual object[] GetData() { return new object[0]; }
879 protected virtual void SetData(object[] a) { }
880 void IFreeze.Freeze(FreezeBuffer fb) {
881 fb.Byte((byte)SerializationCode.ReflectObj);
882 fb.String(GetType().AssemblyQualifiedName);
883 fb.Refs(GetData());
884 }
885
886 internal static ReflectObj Thaw(ThawBuffer tb) {
ead51d9 @sorear Skeleton of -gen-app; use named methods for builtin methods to reduce…
authored
887 string nm = tb.String();
888 if (Backend.cross_level_load)
889 nm = nm.Replace("Run.", "");
890 Type nt = Type.GetType(nm, true);
3f80fca @sorear A procedure for serializing ContextHandler et al
authored
891 ReflectObj n = (ReflectObj)
f0788e8 @sorear Small bugfixes, use correct form of Invoke for .net
authored
892 nt.GetConstructor(new Type[0]).Invoke(new object[0]);
3f80fca @sorear A procedure for serializing ContextHandler et al
authored
893 tb.Register(n);
894 n.SetData(tb.RefsA<object>());
895 return n;
896 }
897 }
8184302 @sorear Start draft of serialization/deserialization code
authored
898 }
Something went wrong with that request. Please try again.