Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Newer
Older
100644 369 lines (308 sloc) 12.037 kB
8184302 @sorear Start draft of serialization/deserialization code
authored
1 using System;
d832c02 @sorear Second draft of serialization code
authored
2 using System.IO;
3 using System.Security.Cryptography;
4 using System.Text;
8184302 @sorear Start draft of serialization/deserialization code
authored
5
cdde7e0 @sorear Notes on four kinds of module scope
authored
6 // Here in Niecza we have four different kinds of unit scopes:
7 //
8 // * COMPILING::UNIT, aka RuntimeUnit: one of these exists for every
9 // call into the compiler, whether eval or module. The REPL will
10 // be considered as if it were eval.
11 //
12 // * Serialization scopes, which are created when compiling modules
13 // or when pre-compiling a main program. During execution there is
14 // no current serialization scope; evals inherit the serialization
15 // scope or lack thereof that was in effect.
16 //
17 // * Assembly scopes, which are an artifact of the CLR and almost align
18 // with precompilation scopes, except that they have to exist always
19 // because methods cannot be created free-floating in CLR 2.x.
20 //
21 // An assembly scope is created for all serialization scopes, and
22 // non-saved anonymous assemblies are created for eval-and-run of
23 // a file and non-BEGIN-time evals.
24 //
25 // * GLOBAL scope is very much like serialization scope except that there
26 // is a "true globals" scope that is used when not serializing.
27
8184302 @sorear Start draft of serialization/deserialization code
authored
28 // This implements "bounded serialization" for Niecza. Unfortunately
29 // the CLR's builtin serialization can't efficiently be made bounded,
30 // and anyway it would be nice if the serialization format could be
31 // transported across backends.
32
33 // TODO: implement a more Storable-like interface.
d832c02 @sorear Second draft of serialization code
authored
34
35 // Note, the serialization subsystem is *NOT* thread safe !
8184302 @sorear Start draft of serialization/deserialization code
authored
36 namespace Niecza.Serialization {
d832c02 @sorear Second draft of serialization code
authored
37 // Information kept on a serialization unit after loading or storing,
38 // but not before storing.
39 class SerUnit {
40 internal string name; // eg "File.Copy"
41 internal byte[] hash; // hash of entire file, filled at write time
42 internal object[] bynum; // objects in unit
43 internal object root; // the RuntimeUnit object
44 internal int nobj;
45 }
8184302 @sorear Start draft of serialization/deserialization code
authored
46
47 // The central feature of *bounded* serialization is that object
48 // registries are kept distinct from the (de)serializer, and can
49 // be shared between serialization runs.
50 class ObjectRegistry {
d832c02 @sorear Second draft of serialization code
authored
51 // TODO: investigate a more specialized representation,
52 // ideally not having to hash as many objects
53 struct ObjRef {
54 SerUnit unit;
55 int id;
8184302 @sorear Start draft of serialization/deserialization code
authored
56 }
d832c02 @sorear Second draft of serialization code
authored
57 Dictionary<object,ObjRef> byref = new Dictionary<object,ObjRef>();
8184302 @sorear Start draft of serialization/deserialization code
authored
58
d832c02 @sorear Second draft of serialization code
authored
59 Dictionary<string,SerUnit> units =
60 new Dictionary<string,SerUnit>();
61
62 static readonly HashAlgorithm hash = SHA256.Create();
63 static readonly string signature = "Niecza-Serialized-Module";
64 static readonly int version = 1;
65
66 // Routines for use by serialization code
67 public bool CheckWriteObject(SerUnit into, object o,
68 out SerUnit lui, out int id) {
69 ObjRef or;
70 if (byref.TryGetValue(o, out or)) {
71 lui = or.unit;
72 id = or.id;
8184302 @sorear Start draft of serialization/deserialization code
authored
73 return true;
d832c02 @sorear Second draft of serialization code
authored
74 }
8184302 @sorear Start draft of serialization/deserialization code
authored
75
d832c02 @sorear Second draft of serialization code
authored
76 if (into.nobj == into.bynum.Length)
77 Array.Resize(ref into.bynum, into.nobj * 2);
8184302 @sorear Start draft of serialization/deserialization code
authored
78
d832c02 @sorear Second draft of serialization code
authored
79 or.unit = into;
80 id = or.id = into.nobj++;
81 into.bynum[id] = o;
82
83 byref[o] = or;
8184302 @sorear Start draft of serialization/deserialization code
authored
84
85 return false;
86 }
d832c02 @sorear Second draft of serialization code
authored
87
88 // Routines for use by compilation manager
89
90 // Loads a single unit from the compiled-data directory.
91 // Will throw a ThawException if a stale reference is encountered
92 // or other data format error.
93 public SerUnit LoadUnit(string name) {
94 SerUnit su;
95
96 // is the unit already loaded?
97 if (units.TryGetValue(name, out su))
98 return su;
99
100 string file = Path.Combine(AppDomain.CurrentDomain.BaseDirectory,
101 name + ".ser");
102 byte[] bytes = File.ReadAllBytes(file);
103
104 su = new SerUnit();
105 su.name = name;
106 su.hash = hash.ComputeHash(bytes);
107
108 ThawBuffer tb = new ThawBuffer(this, unit, bytes, bytes.Length);
109
110 bool success = false;
111 try {
112 string rsig = tb.String();
113 if (rsig != signature)
114 throw new ThawException("signature mismatch loading " + file);
115 int rver = tb.Int();
116 if (rver != version)
117 throw new ThawException("version mismatch loading " + file);
118
119 tb.root = tb.ObjRef();
120 success = true;
121 } finally {
122 // don't leave half-read units in the map
123 if (!success)
124 UnloadUnit(name);
125 }
126
127 return su;
128 }
129
130 // removes a stale unit so a new version can be saved over it.
131 public void UnloadUnit(string name) {
132 SerUnit su = units[name];
133 units.Remove(name);
134
135 for (int i = 0; i < su.nobj; i++)
136 byref.Remove(su.bynum[i]);
137 }
138
139 public SerUnit SaveUnit(string name, IFreeze root) {
140 SerUnit su = new SerUnit();
141 su.name = name;
142 su.root = root;
143
144 if (units.ContainsKey(name))
145 throw new IllegalOperationException();
146
147 bool success = false;
148 string file = Path.Combine(AppDomain.CurrentDomain.BaseDirectory,
149 name + ".ser");
150
151 FreezeBuffer fb = new FreezeBuffer(this, su);
152
153 try {
154 fb.String(signature);
155 fb.Int(version);
156 fb.ObjRef(root);
157
158 byte[] data = fb.GetData();
159 su.hash = hash.ComputeHash(data);
160 File.WriteAllBytes(file, data);
161 success = true;
162 } finally {
163 if (!success)
164 UnloadUnit(name);
165 }
166
167 return su;
168 }
8184302 @sorear Start draft of serialization/deserialization code
authored
169 }
170
171 // One of these codes is written at the beginning of every object ref
172 enum SerializationCode {
d832c02 @sorear Second draft of serialization code
authored
173 // special
174 Null,
175
8184302 @sorear Start draft of serialization/deserialization code
authored
176 // existing objects
177 ForeignRef,
178 SelfRef,
179 NewUnitRef,
180 }
181
182 // An instance of this class is used to serialize serialization units
183 class FreezeBuffer {
184 byte[] data;
185 int wpointer;
186
d832c02 @sorear Second draft of serialization code
authored
187 Dictionary<SerUnit,int> unit_to_offset;
188 int usedunits;
8184302 @sorear Start draft of serialization/deserialization code
authored
189
190 ObjectRegistry reg;
d832c02 @sorear Second draft of serialization code
authored
191 SerUnit unit;
8184302 @sorear Start draft of serialization/deserialization code
authored
192
d832c02 @sorear Second draft of serialization code
authored
193 internal FreezeBuffer(ObjectRegistry reg, SerUnit unit) {
8184302 @sorear Start draft of serialization/deserialization code
authored
194 this.reg = reg;
d832c02 @sorear Second draft of serialization code
authored
195 this.unit = unit;
196 unit_to_offset = new Dictionary<SerUnit,int>();
8184302 @sorear Start draft of serialization/deserialization code
authored
197 data = new byte[256];
198 }
199
200 void Ensure(int ct) {
201 while (ct + wpointer > data.Length)
202 Array.Resize(ref data, data.Length * 2);
203 }
204
205 public void Byte(byte x) {
206 Ensure(1);
207 data[wpointer++] = x;
208 }
209
210 public void Short(short x) {
211 Ensure(2);
212 data[wpointer++] = (byte)(x >> 8);
213 data[wpointer++] = (byte)(x );
214 }
215
216 public void Int(int x) {
217 Ensure(4);
218 data[wpointer++] = (byte)(x >> 24);
219 data[wpointer++] = (byte)(x >> 16);
220 data[wpointer++] = (byte)(x >> 8);
221 data[wpointer++] = (byte)(x );
222 }
223
224 public void Long(long x) {
225 Ensure(8);
226 data[wpointer++] = (byte)(x >> 56);
227 data[wpointer++] = (byte)(x >> 48);
228 data[wpointer++] = (byte)(x >> 40);
229 data[wpointer++] = (byte)(x >> 32);
230 data[wpointer++] = (byte)(x >> 24);
231 data[wpointer++] = (byte)(x >> 16);
232 data[wpointer++] = (byte)(x >> 8);
233 data[wpointer++] = (byte)(x );
234 }
235
236 public void String(string s) {
237 if (s == null) {
238 Int(-1);
239 } else {
240 Int(s.Length);
241 foreach (char ch in s)
242 Short((short)ch);
243 }
244 }
245
d832c02 @sorear Second draft of serialization code
authored
246 // This is the main routine you should call from your Freeze
247 // callbacks to freeze an object
8184302 @sorear Start draft of serialization/deserialization code
authored
248 public void ObjRef(IFreeze o) {
d832c02 @sorear Second draft of serialization code
authored
249 int id;
250 SerUnit altunit;
251 if (o == null) { // null pointers are special
252 Byte((byte)SerializationCode.Null);
253 return;
254 }
255
256 if (reg.CheckWriteObject(unit, o, out altunit, out id)) {
8184302 @sorear Start draft of serialization/deserialization code
authored
257 if (altunit == unit) {
258 Byte((byte)SerializationCode.SelfRef);
259 } else {
d832c02 @sorear Second draft of serialization code
authored
260 int altcode;
261 if (!unit_to_offset.TryGetValue(altunit, out altcode)) {
8184302 @sorear Start draft of serialization/deserialization code
authored
262 Byte((byte)SerializationCode.NewUnitRef);
263 String(reg.UnitName(altunit));
d832c02 @sorear Second draft of serialization code
authored
264 // save the hash too so stale refs can be caught
265 Int(altunit.hash.Length);
266 foreach (byte b in altunit.hash) Byte(b);
267
8184302 @sorear Start draft of serialization/deserialization code
authored
268 unit_to_offset[altunit] = usedunits++;
269 } else {
270 Byte((byte)SerializationCode.ForeignRef);
d832c02 @sorear Second draft of serialization code
authored
271 Int(altcode);
8184302 @sorear Start draft of serialization/deserialization code
authored
272 }
273 }
274 Int((int)id);
275 } else {
d832c02 @sorear Second draft of serialization code
authored
276 // must take responsibility for saving the tag
8184302 @sorear Start draft of serialization/deserialization code
authored
277 o.Freeze(this);
278 }
279 }
280 }
281
282 // Note that this interface only handles freezing - thaw is done using
283 // a switch statement.
284 interface IFreeze {
285 void Freeze(FreezeBuffer fb);
286 }
287
288 class ThawBuffer {
289 byte[] data;
290 int dlen;
291 int rpointer;
292 ObjectRegistry reg;
293
d832c02 @sorear Second draft of serialization code
authored
294 SerUnit[] unit_map = new SerUnit[8];
8184302 @sorear Start draft of serialization/deserialization code
authored
295 int refed_units;
d832c02 @sorear Second draft of serialization code
authored
296 SerUnit unit;
8184302 @sorear Start draft of serialization/deserialization code
authored
297
d832c02 @sorear Second draft of serialization code
authored
298 internal ThawBuffer(ObjectRegistry reg, SerUnit unit,
299 byte[] data, int dlen) {
8184302 @sorear Start draft of serialization/deserialization code
authored
300 this.data = data;
301 this.dlen = dlen;
302 this.reg = reg;
d832c02 @sorear Second draft of serialization code
authored
303 this.unit = unit;
8184302 @sorear Start draft of serialization/deserialization code
authored
304 }
305
306 object ObjRef() {
307 var tag = (SerializationCode)Byte();
308 int i, j;
309 switch(tag) {
d832c02 @sorear Second draft of serialization code
authored
310 case SerializationCode.Null:
311 return null;
8184302 @sorear Start draft of serialization/deserialization code
authored
312 case SerializationCode.SelfRef:
313 i = Int();
d832c02 @sorear Second draft of serialization code
authored
314 return unit.bynum[i];
8184302 @sorear Start draft of serialization/deserialization code
authored
315 case SerializationCode.ForeignRef:
316 i = Int();
317 j = Int();
d832c02 @sorear Second draft of serialization code
authored
318 return unit_map[i].bynum[j];
8184302 @sorear Start draft of serialization/deserialization code
authored
319 case SerializationCode.NewUnitRef:
d832c02 @sorear Second draft of serialization code
authored
320 return LoadNewUnit();
8184302 @sorear Start draft of serialization/deserialization code
authored
321 default:
322 throw new ThawException("unexpected object tag" + (byte)tag);
323 }
324 }
d832c02 @sorear Second draft of serialization code
authored
325
326 object LoadNewUnit() {
327 string name = String();
328 if (refed_units == unit_map.Length)
329 Array.Resize(ref unit_map, refed_units * 2);
330
331 SerUnit su = reg.LoadUnit(name);
332 unit_map[refed_units++] = su;
333
334 byte[] hash = Bytes(su.hash.Length);
335
336 for (int i = 0; i < hash.Length; i++)
337 if (hash[i] != su.hash[i])
338 goto badhash;
339
340 int ix = Int();
341 return su.bynum[ix];
342
343 badhash:
344 StringBuilder sb = new StringBuilder();
345 sb.AppendFormat("Hash mismatch for unit {0} referenced from {1}",
346 su.name, unit.name);
347
348 sb.Append(", wanted ");
349 foreach (byte b in hash)
350 sb.AppendFormat("{0:X2}", b);
351
352 sb.Append(", got ");
353 foreach (byte b in su.hash)
354 sb.AppendFormat("{0:X2}", b);
355
356 throw new ThawException(sb.ToString());
357 }
358 }
359
360 // Thrown to indicate data format problems in the serialized stream
361 // Not necessarily bugs; could also indicate stale files, including
362 // cases where the data format is changed and cases where a depended
363 // file was recreated
364 class ThawException : Exception {
365 public ThawException(string s) : base(s) { }
366 public ThawException() : base() { }
8184302 @sorear Start draft of serialization/deserialization code
authored
367 }
368 }
Something went wrong with that request. Please try again.