Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
tree: bbf7bbd230
Fetching contributors…

Cannot retrieve contributors at this time

899 lines (780 sloc) 29.755 kb
using System;
using System.IO;
using System.Security.Cryptography;
using System.Collections.Generic;
using System.Reflection;
using System.Text;
using Niecza.CLRBackend;
// Here in Niecza we have four different kinds of unit scopes:
//
// * COMPILING::UNIT, aka RuntimeUnit: one of these exists for every
// call into the compiler, whether eval or module. The REPL will
// be considered as if it were eval.
//
// * Serialization scopes, which are created when compiling modules
// or when pre-compiling a main program. During execution there is
// no current serialization scope; evals inherit the serialization
// scope or lack thereof that was in effect.
//
// * Assembly scopes, which are an artifact of the CLR and almost align
// with precompilation scopes, except that they have to exist always
// because methods cannot be created free-floating in CLR 2.x.
//
// An assembly scope is created for all serialization scopes, and
// non-saved anonymous assemblies are created for eval-and-run of
// a file and non-BEGIN-time evals.
//
// * GLOBAL scope is very much like serialization scope except that there
// is a "true globals" scope that is used when not serializing.
// This implements "bounded serialization" for Niecza. Unfortunately
// the CLR's builtin serialization can't efficiently be made bounded,
// and anyway it would be nice if the serialization format could be
// transported across backends.
// TODO: implement a more Storable-like interface.
// Note, the serialization subsystem is *NOT* thread safe !
namespace Niecza.Serialization {
// Information kept on a serialization unit after loading or storing,
// but not before storing.
class SerUnit {
internal string name; // eg "File.Copy"
internal byte[] hash; // hash of entire file, filled at write time
internal object[] bynum = new object[8]; // objects in unit
internal object root; // the RuntimeUnit object
internal int nobj; // = 0
internal bool fake; // true for CLR reference units
}
// The central feature of *bounded* serialization is that object
// registries are kept distinct from the (de)serializer, and can
// be shared between serialization runs.
class ObjectRegistry {
// TODO: investigate a more specialized representation,
// ideally not having to hash as many objects
struct ObjRef {
public SerUnit unit;
public int id;
}
Dictionary<object,ObjRef> byref = new Dictionary<object,ObjRef>();
Dictionary<string,SerUnit> units =
new Dictionary<string,SerUnit>();
internal static HashAlgorithm NewHash() { return new SHA256Managed(); }
internal Dictionary<string,Dictionary<string,MethodInfo>> methods =
new Dictionary<string,Dictionary<string,MethodInfo>>();
internal Dictionary<string,object> instances =
new Dictionary<string,object>();
static readonly string signature = "Niecza-Serialized-Module";
static readonly int version = 30;
// Routines for use by serialization code
public bool CheckWriteObject(SerUnit into, object o,
out SerUnit lui, out int id) {
ObjRef or;
if (byref.TryGetValue(o, out or)) {
lui = or.unit;
id = or.id;
return true;
}
if (into.nobj == into.bynum.Length)
Array.Resize(ref into.bynum, into.nobj * 2);
or.unit = lui = into;
id = or.id = into.nobj++;
into.bynum[id] = o;
byref[o] = or;
return false;
}
// Undo previous method...
public void DeleteLastObject(SerUnit into) {
byref.Remove(into.bynum[--into.nobj]);
into.bynum[into.nobj] = null;
}
public void RegisterThawed(SerUnit into, object o) {
ObjRef or;
if (into.nobj == into.bynum.Length)
Array.Resize(ref into.bynum, into.nobj * 2);
or.unit = into;
or.id = into.nobj++;
into.bynum[or.id] = o;
byref[o] = or;
}
// Routines for use by compilation manager
public void InstallFakeUnit(string name, params object[] items) {
SerUnit su = new SerUnit();
su.fake = true;
su.name = name;
su.bynum = items;
su.hash = new byte[0];
units[name] = su;
ObjRef or;
or.unit = su;
for (or.id = 0; or.id < items.Length; or.id++)
byref[items[or.id]] = or;
}
// Loads a single unit from the compiled-data directory.
// Will throw a ThawException if a stale reference is encountered
// or other data format error.
public SerUnit LoadUnit(string name, bool fake = false) {
SerUnit su;
// is the unit already loaded?
if (units.TryGetValue(name, out su))
return su;
if (fake) {
if (name.StartsWith("CLR,"))
CLRWrapperProvider.LoadWrapper(Compartment.Top, name.Substring(4));
else
throw new ThawException("No handler for fake unit name " + name);
return units[name];
}
string file = Path.Combine(Backend.obj_dir, Backend.prefix +
name.Replace("::",".") + ".ser");
byte[] bytes = File.ReadAllBytes(file);
su = new SerUnit();
su.name = name;
su.hash = NewHash().ComputeHash(bytes);
ThawBuffer tb = new ThawBuffer(Compartment.Top, this, su, bytes);
units[name] = su;
bool success = false;
try {
string rsig = tb.String();
if (rsig != signature)
throw new ThawException("signature mismatch loading " + file);
int rver = tb.Int();
if (rver != version)
throw new ThawException("version mismatch loading " + file);
su.root = tb.ObjRef();
tb.RunFixups();
success = true;
} finally {
// don't leave half-read units in the map
if (!success)
UnloadUnit(name);
}
return su;
}
// removes a stale unit so a new version can be saved over it.
public void UnloadUnit(string name) {
if (!units.ContainsKey(name))
return;
SerUnit su = units[name];
units.Remove(name);
for (int i = 0; i < su.nobj; i++)
byref.Remove(su.bynum[i]);
}
public SerUnit SaveUnit(string name, IFreeze root) {
SerUnit su = new SerUnit();
su.name = name;
su.root = root;
if (units.ContainsKey(name))
throw new InvalidOperationException("unit " +name+ " exists");
bool success = false;
string file = Path.Combine(Backend.obj_dir, Backend.prefix +
name.Replace("::",".") + ".ser");
FreezeBuffer fb = new FreezeBuffer(this, su);
units[name] = su;
try {
fb.String(signature);
fb.Int(version);
fb.ObjRef(root);
byte[] data = fb.GetData();
su.hash = NewHash().ComputeHash(data);
File.WriteAllBytes(file, data);
success = true;
} finally {
if (!success)
UnloadUnit(name);
}
return su;
}
}
// One of these codes is written at the beginning of every object ref
enum SerializationCode : byte {
// special
Null,
// existing objects
ForeignRef,
SelfRef,
NewUnitRef,
FakeUnitRef,
// types of new object
RuntimeUnit,
SubInfo,
STable,
StashEnt,
Rat,
FatRat,
Complex,
BigInteger,
VarDeque,
VarHash,
DispatchEnt,
RxFrame,
P6how,
ReflectObj,
CC,
AltInfo,
Signature,
Parameter,
// types of P6any-reified object
P6opaque, // eventually let's specialize this
Frame,
Cursor,
// miscellany - keep these in same order as FallbackFreeze
String,
ArrP6any,
ArrVariable,
ArrString,
ArrCC,
Boolean,
Int,
Double,
Type,
// variables
RWVariable, // allow 2 for flag
RWVariable_1,
ListVariable,
SubstrLValue,
TiedVariable,
Blackhole,
// vivification hooks
SubViviHook,
ArrayViviHook,
NewArrayViviHook,
HashViviHook,
NewHashViviHook,
// Longest-token automaton descriptors
LADNone, // no-args
LADNull,
LADDot,
LADDispatcher,
LADImp,
LADStr, // string
LADStrNoCase,
LADMethod,
LADParam,
LADQuant, // byte,LAD,LAD
LADSequence, // LAD[]
LADAny,
LADCC, // CC
}
// An instance of this class is used to serialize serialization units
public class FreezeBuffer {
byte[] data;
int wpointer;
Dictionary<SerUnit,int> unit_to_offset;
int usedunits;
ObjectRegistry reg;
SerUnit unit;
internal FreezeBuffer(ObjectRegistry reg, SerUnit unit) {
if (reg == null || unit == null)
throw new ArgumentNullException();
this.reg = reg;
this.unit = unit;
unit_to_offset = new Dictionary<SerUnit,int>();
data = new byte[256];
}
internal byte[] GetData() {
byte[] ret = new byte[wpointer];
Array.Copy(data, ret, ret.Length);
return ret;
}
void Ensure(int ct) {
while (ct + wpointer > data.Length)
Array.Resize(ref data, data.Length * 2);
}
public void Byte(byte x) {
Ensure(1);
data[wpointer++] = x;
}
public void Long(long x) {
//Console.WriteLine("Saving {0} at {1}", x, wpointer);
Ensure(10);
while (true) {
if (x >= -64 && x <= 63) {
data[wpointer++] = (byte) (127 & (byte)x);
break;
} else {
data[wpointer++] = (byte) (128 | (byte)x);
x >>= 7;
}
}
}
public void ULong(ulong x) {
//Console.WriteLine("Saving {0} at {1}", x, wpointer);
Ensure(10);
while (true) {
if (x <= 127) {
data[wpointer++] = (byte) (127 & (byte)x);
break;
} else {
data[wpointer++] = (byte) (128 | (byte)x);
x >>= 7;
}
}
}
public void Short(short x) { Long(x); }
public void Int(int x) { Long(x); }
public void Double(double x) {
Long(BitConverter.DoubleToInt64Bits(x));
}
public void String(string s) {
if (s == null) {
Int(-1);
} else {
Int(s.Length);
foreach (char ch in s)
ULong((ulong)ch);
}
}
public void Strings(string[] s) {
if (s == null) Int(-1);
else {
Int(s.Length);
foreach (string ch in s) String(ch);
}
}
public void Ints(int[] s) {
if (s == null) {
Int(-1);
} else {
Int(s.Length);
foreach (int ch in s)
Int(ch);
}
}
public void Refs<T> (T[] x) {
if (x == null) {
Int(-1);
} else {
Int(x.Length);
foreach (T y in x)
ObjRef(y);
}
}
public void Refs<T> (IList<T> x) {
if (x == null) {
Int(-1);
} else {
Int(x.Count);
foreach (T y in x)
ObjRef(y);
}
}
public void Ephemeralize() {
Byte((byte)SerializationCode.Null);
reg.DeleteLastObject(unit);
}
// This is the main routine you should call from your Freeze
// callbacks to freeze an object
static int depth;
public void ObjRef(object o) {
int id;
SerUnit altunit;
if (Config.SerTrace) {
Console.WriteLine("{0}Saving {1} at {2}:", new string(' ',depth), o, wpointer);
}
if (o == null) { // null pointers are special
Byte((byte)SerializationCode.Null);
return;
}
if (reg.CheckWriteObject(unit, o, out altunit, out id)) {
if (altunit == unit) {
Byte((byte)SerializationCode.SelfRef);
} else {
int altcode;
if (!unit_to_offset.TryGetValue(altunit, out altcode)) {
Byte(altunit.fake ?
(byte)SerializationCode.FakeUnitRef :
(byte)SerializationCode.NewUnitRef);
String(altunit.name);
// save the hash too so stale refs can be caught
foreach (byte b in altunit.hash) Byte(b);
unit_to_offset[altunit] = usedunits++;
} else {
Byte((byte)SerializationCode.ForeignRef);
Int(altcode);
}
}
Int((int)id);
} else {
// must take responsibility for saving the tag
IFreeze f = o as IFreeze;
if (Config.SerTrace) depth++;
if (f != null) {
f.Freeze(this);
} else {
FallbackFreeze(o);
}
if (Config.SerTrace) depth--;
}
}
[Immutable]
internal static Type[] boxTypes = new Type[] {
null, typeof(Rat), typeof(FatRat), typeof(Complex),
typeof(double), typeof(int), typeof(string), typeof(VarHash),
typeof(Variable[]), typeof(VarDeque), typeof(STable),
typeof(BigInteger),
};
[Immutable]
internal static Func<P6opaque>[] boxCreate = new Func<P6opaque>[] {
P6opaque.Create, BoxObject<Rat>.Create, BoxObject<FatRat>.Create,
BoxObject<Complex>.Create, BoxObject<double>.Create,
BoxObject<int>.Create, BoxObject<string>.Create,
BoxObject<VarHash>.Create, BoxObject<Variable[]>.Create,
BoxObject<VarDeque>.Create, BoxObject<STable>.Create,
BoxObject<BigInteger>.Create,
};
[Immutable]
static Type[] anyTypes = new Type[] {
typeof(string), typeof(P6any[]), typeof(Variable[]),
typeof(string[]), typeof(CC[]),
typeof(bool), typeof(int), typeof(double), typeof(Type),
};
void FallbackFreeze(object o) {
int ix = 0;
Type t = o.GetType();
while (ix != anyTypes.Length && anyTypes[ix] != t) ix++;
Byte((byte)(((int)SerializationCode.String) + ix));
switch(ix) {
case 0:
String((string)o);
break;
case 1:
Refs((P6any[])o);
break;
case 2:
Refs((Variable[])o);
break;
case 3:
Refs((string[])o);
break;
case 4:
Refs((CC[])o);
break;
case 5:
Byte((byte)((bool)o ? 1 : 0));
break;
case 6:
Int((int)o);
break;
case 7:
Double((double)o);
break;
case 8:
String(((Type)o).AssemblyQualifiedName);
break;
default:
throw new NotImplementedException(t.FullName);
}
}
}
// Note that this interface only handles freezing - thaw is done using
// a switch statement.
public interface IFreeze {
void Freeze(FreezeBuffer fb);
}
// implement this if you need to copy in data from other objects, &c
interface IFixup {
void Fixup();
}
class ThawBuffer {
byte[] data;
int rpointer;
internal readonly ObjectRegistry reg;
internal readonly Compartment setting;
SerUnit[] unit_map = new SerUnit[8];
int refed_units;
SerUnit unit;
List<IFixup> fixups_needed = new List<IFixup>();
List<object> revalidate = new List<object>();
public Type type;
internal ThawBuffer(Compartment setting, ObjectRegistry reg, SerUnit unit, byte[] data) {
this.data = data;
this.setting = setting;
this.reg = reg;
this.unit = unit;
}
internal void RunFixups() {
P6how.BulkRevalidate(revalidate);
foreach (IFixup f in fixups_needed)
f.Fixup();
fixups_needed.Clear();
}
internal void PushFixup(IFixup f) {
fixups_needed.Add(f);
}
internal void PushRevalidate(STable f) {
revalidate.Add(f);
}
public byte Byte() { return data[rpointer++]; }
public long Long() {
int shift = 0;
long accum = 0;
while (true) {
byte b = Byte();
accum |= (((long)(b & 127)) << shift);
shift += 7;
if ((b & 128) == 0) {
if ((b & 64) != 0 && shift < 64) {
accum |= ((-1L) << shift);
}
//Console.WriteLine("Read {0} end {1}", accum, rpointer);
return accum;
}
}
}
public ulong ULong() {
int shift = 0;
ulong accum = 0;
while (true) {
byte b = Byte();
accum |= (((ulong)(b & 127)) << shift);
shift += 7;
if ((b & 128) == 0) {
//Console.WriteLine("Read {0} end {1}", accum, rpointer);
return accum;
}
}
}
public short Short() {
return checked((short)Long());
}
public int Int() {
return checked((int)Long());
}
public double Double() {
return BitConverter.Int64BitsToDouble(Long());
}
public string String() {
int l = Int();
if (l < 0) return null;
char[] cb = new char[l];
for (int i = 0; i < l; i++)
cb[i] = (char)ULong();
return new string(cb);
}
public byte[] Bytes(int k) {
byte[] buf = new byte[k];
for (int i = 0; i < k; i++)
buf[i] = Byte();
return buf;
}
public List<T> RefsL<T>() where T : class {
int ct = Int();
if (ct < 0) return null;
List<T> ret = new List<T>();
for (int i = 0; i < ct; i++)
ret.Add((T) ObjRef());
return ret;
}
public int[] Ints() {
int ct = Int();
if (ct < 0) return null;
int[] ret = new int[ct];
for (int i = 0; i < ct; i++) ret[i] = Int();
return ret;
}
public string[] Strings() {
int ct = Int();
if (ct < 0) return null;
string[] ret = new string[ct];
for (int i = 0; i < ct; i++) ret[i] = String();
return ret;
}
public T[] RefsA<T>() where T : class {
int ct = Int();
if (ct < 0) return null;
T[] ret = new T[ct];
for (int i = 0; i < ct; i++)
ret[i] = (T) ObjRef();
return ret;
}
// used from ObjRef only so guaranteed non-null
T[] RefsARegister<T>() where T : class {
int ct = Int();
T[] ret = new T[ct];
Register(ret);
for (int i = 0; i < ct; i++)
ret[i] = (T) ObjRef();
return ret;
}
public object ObjRef() {
var tag = (SerializationCode)Byte();
if (Config.SerTrace)
Console.WriteLine("Reading {0} from {1}...", tag, rpointer-1);
int i, j;
switch(tag) {
case SerializationCode.Null:
return null;
case SerializationCode.ForeignRef:
i = Int();
j = Int();
return unit_map[i].bynum[j];
case SerializationCode.SelfRef:
i = Int();
return unit.bynum[i];
case SerializationCode.NewUnitRef:
return LoadNewUnit(false);
case SerializationCode.FakeUnitRef:
return LoadNewUnit(true);
case SerializationCode.RuntimeUnit:
return RuntimeUnit.Thaw(this);
case SerializationCode.SubInfo:
return SubInfo.Thaw(this);
case SerializationCode.STable:
return STable.Thaw(this);
case SerializationCode.StashEnt:
return StashEnt.Thaw(this);
case SerializationCode.Rat:
return Rat.Thaw(this);
case SerializationCode.FatRat:
return FatRat.Thaw(this);
case SerializationCode.Complex:
return Complex.Thaw(this);
case SerializationCode.BigInteger:
return BigInteger.Thaw(this);
case SerializationCode.VarDeque:
return VarDeque.Thaw(this);
case SerializationCode.VarHash:
return VarHash.Thaw(this);
case SerializationCode.DispatchEnt:
return DispatchEnt.Thaw(this);
//case SerializationCode.RxFrame:
// return RxFrame.Thaw(this);
case SerializationCode.P6how:
return P6how.Thaw(this);
case SerializationCode.CC:
return CC.Thaw(this);
case SerializationCode.AltInfo:
return AltInfo.Thaw(this);
case SerializationCode.Signature:
return Signature.Thaw(this);
case SerializationCode.Parameter:
return Parameter.Thaw(this);
case SerializationCode.ReflectObj:
return ReflectObj.Thaw(this);
case SerializationCode.P6opaque:
return P6opaque.Thaw(this);
case SerializationCode.Frame:
return Frame.Thaw(this);
//Cursor,
case SerializationCode.String:
return Register(String());
case SerializationCode.ArrP6any:
return RefsARegister<P6any>();
case SerializationCode.ArrVariable:
return RefsARegister<Variable>();
case SerializationCode.ArrString:
return RefsARegister<string>();
case SerializationCode.ArrCC:
return RefsARegister<CC>();
case SerializationCode.Boolean:
return Register(Byte() != 0);
case SerializationCode.Int:
return Register(Int());
case SerializationCode.Double:
return Register(Double());
case SerializationCode.Type:
return Register(Type.GetType(String(), true));
case SerializationCode.ListVariable:
return ListVariable.Thaw(this);
case SerializationCode.RWVariable:
case SerializationCode.RWVariable_1:
return RWVariable.Thaw(this,
(int)tag - (int)SerializationCode.RWVariable);
case SerializationCode.SubstrLValue:
return SubstrLValue.Thaw(this);
case SerializationCode.TiedVariable:
return TiedVariable.Thaw(this);
case SerializationCode.Blackhole:
return Builtins.Blackhole.Thaw(this);
case SerializationCode.SubViviHook:
return SubViviHook.Thaw(this);
case SerializationCode.ArrayViviHook:
return ArrayViviHook.Thaw(this);
case SerializationCode.NewArrayViviHook:
return NewArrayViviHook.Thaw(this);
case SerializationCode.HashViviHook:
return HashViviHook.Thaw(this);
case SerializationCode.NewHashViviHook:
return NewHashViviHook.Thaw(this);
case SerializationCode.LADNone:
return Register(new LADNone());
case SerializationCode.LADNull:
return Register(new LADNull());
case SerializationCode.LADDot:
return Register(new LADDot());
case SerializationCode.LADDispatcher:
return Register(new LADDispatcher());
case SerializationCode.LADImp:
return Register(new LADImp());
case SerializationCode.LADStr:
return LADStr.Thaw(this);
case SerializationCode.LADStrNoCase:
return LADStrNoCase.Thaw(this);
case SerializationCode.LADMethod:
return LADMethod.Thaw(this);
case SerializationCode.LADParam:
return LADParam.Thaw(this);
case SerializationCode.LADQuant:
return LADQuant.Thaw(this);
case SerializationCode.LADSequence:
return LADSequence.Thaw(this);
case SerializationCode.LADAny:
return LADAny.Thaw(this);
case SerializationCode.LADCC:
return LADCC.Thaw(this);
default:
throw new ThawException("unexpected object tag " + tag);
}
}
// call this when thawing any new object
internal object Register(object o) {
reg.RegisterThawed(unit, o);
return o;
}
object LoadNewUnit(bool fake) {
string name = String();
if (refed_units == unit_map.Length)
Array.Resize(ref unit_map, refed_units * 2);
SerUnit su = reg.LoadUnit(name, fake);
unit_map[refed_units++] = su;
byte[] hash = Bytes(su.hash.Length);
for (int i = 0; i < hash.Length; i++)
if (hash[i] != su.hash[i])
goto badhash;
int ix = Int();
return su.bynum[ix];
badhash:
throw new ThawException(string.Format("Hash mismatch for " +
"unit {0} referenced from {1}, wanted {2}, got {3}",
su.name, unit.name, Utils.HashToStr(unit.hash),
Utils.HashToStr(su.hash)));
}
}
// Thrown to indicate data format problems in the serialized stream
// Not necessarily bugs; could also indicate stale files, including
// cases where the data format is changed and cases where a depended
// file was recreated
class ThawException : Exception {
public ThawException(string s) : base(s) { }
public ThawException() : base() { }
}
public class ReflectObj : IFreeze {
internal Compartment setting = Compartment.Top;
protected virtual object[] GetData() { return new object[0]; }
protected virtual void SetData(object[] a) { }
void IFreeze.Freeze(FreezeBuffer fb) {
fb.Byte((byte)SerializationCode.ReflectObj);
fb.String(GetType().AssemblyQualifiedName);
fb.Refs(GetData());
}
internal static ReflectObj Thaw(ThawBuffer tb) {
string nm = tb.String();
if (Backend.cross_level_load)
nm = nm.Replace("Run.", "");
Type nt = Type.GetType(nm, true);
ReflectObj n = (ReflectObj)
nt.GetConstructor(new Type[0]).Invoke(new object[0]);
tb.Register(n);
n.SetData(tb.RefsA<object>());
return n;
}
}
}
Jump to Line
Something went wrong with that request. Please try again.