Add second-pass objects and translation.

Author: progval

src/lib.rs

@@ -1,3 +1,37 @@
 mod marshal;
-mod processor;
+mod objects;
+//mod processor;
 
+use std::fmt;
+use std::io;
+
+#[derive(Debug)]
+pub enum InterpreterError {
+    Io(io::Error),
+    Unmarshal(marshal::decode::UnmarshalError),
+}
+
+impl fmt::Display for InterpreterError {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match *self {
+            InterpreterError::Io(ref e) => write!(f, "I/O error:").and_then(|_| e.fmt(f)),
+            InterpreterError::Unmarshal(ref e) => write!(f, "Unmarshal error:").and_then(|_| e.fmt(f)),
+        }
+    }
+}
+
+pub fn run_module<R: io::Read>(reader: &mut R) -> Result<(), InterpreterError> {
+    let mut buf = [0; 12];
+    try!(reader.read_exact(&mut buf).map_err(InterpreterError::Io));
+    // TODO: do something with the content of the buffer
+    let mut store = objects::ObjectStore::new();
+    let module = try!(marshal::read_object(reader, &mut store).map_err(InterpreterError::Unmarshal));
+    //processor::run_code_object(module, &mut references);
+    Ok(())
+}
+
+#[test]
+fn test_hello_world() {
+    let mut reader: &[u8] = b"\xee\x0c\r\n\x15j\nW\x15\x00\x00\x00\xe3\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00@\x00\x00\x00s\x0e\x00\x00\x00e\x00\x00d\x00\x00\x83\x01\x00\x01d\x01\x00S)\x02z\x0bHello worldN)\x01\xda\x05print\xa9\x00r\x02\x00\x00\x00r\x02\x00\x00\x00\xfa\x0b/tmp/foo.py\xda\x08<module>\x01\x00\x00\x00s\x00\x00\x00\x00";
+    run_module(&mut reader);
+}

src/marshal/common.rs

@@ -19,6 +19,7 @@ pub struct Code {
     pub lnotab: Object,
 }
 
+/// Temporary object used only during the unmarshalling pass
 #[derive(PartialEq)]
 #[derive(Debug)]
 pub enum Object {

src/marshal/unmarshal.rs renamed to src/marshal/decode.rs

@@ -73,39 +73,40 @@ fn read_unicode_string<R: io::Read>(reader: &mut R, size: usize) -> Result<Strin
 }
 
 /// Read an arbitrary number of contiguous marshal objects
-fn read_objects<R: io::Read>(reader: &mut R, references: &mut Vec<Object>, size: usize) -> Result<Vec<Object>, UnmarshalError> {
+fn read_tmp_objects<R: io::Read>(reader: &mut R, references: &mut Vec<Object>, size: usize) -> Result<Vec<Object>, UnmarshalError> {
     let mut vector = Vec::<Object>::new();
     vector.reserve(size);
     for _ in 0..size {
-        let object = try!(read_object(reader, references));
+        let object = try!(read_tmp_object(reader, references));
         vector.push(object);
     };
     Ok(vector)
 }
 
-/// Read objects and build an other object containing them.
+/// Read temporary marshal objects and build an other object containing them.
 /// If the flag is true, add this object to the vector of objects before reading its content
 /// (required, as the order of objects matter for references).
 macro_rules! build_container {
     ( $reader:expr, $references:ident, $container:expr, $size:expr, $flag:expr) => {{
         if $flag {
             let index = $references.len() as u32; // TODO: overflow check
             $references.push(Object::Hole);
-            let objects = try!(read_objects($reader, $references, $size));
+            let objects = try!(read_tmp_objects($reader, $references, $size));
             $references[index as usize] = $container(objects); // TODO: overflow check
             (false, Object::Ref(index))
         }
         else {
-            let objects = try!(read_objects($reader, $references, $size));
+            let objects = try!(read_tmp_objects($reader, $references, $size));
             (false, $container(objects))
         }
     }}
 }
 
-/// Read an object, whose type is known from the first byte. If it is a container, read its content too.
+/// Read an tmporary marshal object, whose type is known from the first byte.
+/// If it is a container, read its content too.
 /// If the first bit is 1 and the marshal protocol allows the type to be referenced,
 /// add it to the list of references too.
-pub fn read_object<R: io::Read>(reader: &mut R, references: &mut Vec<Object>) -> Result<Object, UnmarshalError> {
+pub fn read_tmp_object<R: io::Read>(reader: &mut R, references: &mut Vec<Object>) -> Result<Object, UnmarshalError> {
     let byte = read_byte!(reader);
     let flag = byte & 0b10000000 != 0;
     let opcode = byte & 0b01111111;
@@ -164,16 +165,16 @@ pub fn read_object<R: io::Read>(reader: &mut R, references: &mut Vec<Object>) ->
                 nlocals: try!(read_long(reader)),
                 stacksize: try!(read_long(reader)),
                 flags: try!(read_long(reader)),
-                code: try!(read_object(reader, references)),
-                consts: try!(read_object(reader, references)),
-                names: try!(read_object(reader, references)),
-                varnames: try!(read_object(reader, references)),
-                freevars: try!(read_object(reader, references)),
-                cellvars: try!(read_object(reader, references)),
-                filename: try!(read_object(reader, references)),
-                name: try!(read_object(reader, references)),
+                code: try!(read_tmp_object(reader, references)),
+                consts: try!(read_tmp_object(reader, references)),
+                names: try!(read_tmp_object(reader, references)),
+                varnames: try!(read_tmp_object(reader, references)),
+                freevars: try!(read_tmp_object(reader, references)),
+                cellvars: try!(read_tmp_object(reader, references)),
+                filename: try!(read_tmp_object(reader, references)),
+                name: try!(read_tmp_object(reader, references)),
                 firstlineno: try!(read_long(reader)),
-                lnotab: try!(read_object(reader, references)), // TODO: decode this
+                lnotab: try!(read_tmp_object(reader, references)), // TODO: decode this
             };
 
             let object = Object::Code(Box::new(code));
@@ -195,13 +196,13 @@ macro_rules! assert_unmarshal {
     ( $expected_obj:expr, $bytecode:expr) => {{
         let mut reader: &[u8] = $bytecode;
         let mut refs = Vec::new();
-        let obj = read_object(&mut reader, &mut refs).unwrap();
+        let obj = read_tmp_object(&mut reader, &mut refs).unwrap();
         assert_eq!($expected_obj, obj);
     }};
     ( $expected_obj:expr, $expected_refs:expr, $bytecode:expr) => {{
         let mut reader: &[u8] = $bytecode;
         let mut refs = Vec::new();
-        let obj = read_object(&mut reader, &mut refs).unwrap();
+        let obj = read_tmp_object(&mut reader, &mut refs).unwrap();
         assert_eq!($expected_obj, obj);
         assert_eq!($expected_refs, refs);
     }};

src/marshal/mod.rs

@@ -1,2 +1,60 @@
 pub mod common;
-pub mod unmarshal;
+pub mod decode;
+
+use std::io;
+use std::collections::HashSet;
+use super::objects::{Code, ObjectContent, Object, ObjectRef, ObjectStore};
+use self::common::Object as MarshalObject;
+
+macro_rules! translate_vector {
+    ( $e:expr, $map:ident, $store:ident ) => { $e.into_iter().map(|o| translate_object(o, $map, $store)).collect() }
+}
+
+fn translate_object(marshal_object: MarshalObject, translation_map: &Vec<ObjectRef>, store: &mut ObjectStore) -> ObjectRef {
+    match marshal_object {
+        MarshalObject::Hole => panic!("Remaining hole."),
+        MarshalObject::None => store.allocate(ObjectContent::None),
+        MarshalObject::False => store.allocate(ObjectContent::False),
+        MarshalObject::True => store.allocate(ObjectContent::True),
+        MarshalObject::Int(i) => store.allocate(ObjectContent::Int(i)),
+        MarshalObject::String(s) => store.allocate(ObjectContent::String(s)),
+        MarshalObject::Bytes(v) => store.allocate(ObjectContent::Bytes(v)),
+        MarshalObject::Tuple(v) => {
+            let v = translate_vector!(v, translation_map, store);
+            store.allocate(ObjectContent::Tuple(v))
+        },
+        MarshalObject::List(v) => {
+            let v = translate_vector!(v, translation_map, store);
+            store.allocate(ObjectContent::List(v))
+        },
+        MarshalObject::Set(v) => {
+            let v = translate_vector!(v, translation_map, store);
+            store.allocate(ObjectContent::Set(v))
+        },
+        MarshalObject::FrozenSet(v) => {
+            let v = translate_vector!(v, translation_map, store);
+            store.allocate(ObjectContent::FrozenSet(v))
+        },
+        MarshalObject::Code(c) => {
+            let code = translate_object(c.code, translation_map, store);
+            store.allocate(ObjectContent::Code(Code { code: code})) // TODO: more fields
+        },
+        MarshalObject::Ref(i) => translation_map.get(i as usize).unwrap().clone(), // TODO: overflow check
+    }
+}
+
+fn translate_objects(marshal_object: MarshalObject, references: Vec<MarshalObject>, store: &mut ObjectStore) -> ObjectRef {
+    let mut translation_map = Vec::new();
+    translation_map.reserve(references.len());
+    for obj in references {
+        let obj = translate_object(obj, &translation_map, store);
+        translation_map.push(obj);
+    }
+    translate_object(marshal_object, &translation_map, store)
+}
+
+pub fn read_object<R: io::Read>(reader: &mut R, store: &mut ObjectStore) -> Result<ObjectRef, decode::UnmarshalError> {
+    let mut references = Vec::new();
+    let marshal_object = try!(decode::read_tmp_object(reader, &mut references));
+    Ok(translate_objects(marshal_object, references, store))
+}

src/objects/mod.rs

@@ -0,0 +1,62 @@
+
+#[derive(Debug)]
+pub struct Code {/*
+    pub argcount: u32,
+    pub kwonlyargcount: u32,
+    pub nlocals: u32,
+    pub stacksize: u32,
+    pub flags: u32,*/
+    pub code: ObjectRef,/*
+    pub consts: Object,
+    pub names: Object,
+    pub varnames: Object,
+    pub freevars: Object,
+    pub cellvars: Object,
+    pub filename: Object,
+    pub name: Object,
+    pub firstlineno: u32,
+    pub lnotab: Object,*/
+}
+
+#[derive(Debug)]
+pub enum ObjectContent {
+    None,
+    True,
+    False,
+    Int(u32),
+    String(::std::string::String),
+    Tuple(Vec<ObjectRef>),
+    List(Vec<ObjectRef>),
+    Code(Code),
+    Set(Vec<ObjectRef>),
+    FrozenSet(Vec<ObjectRef>),
+    Bytes(Vec<u8>),
+}
+
+#[derive(Debug)]
+pub struct Object {
+    content: ObjectContent,
+}
+
+#[derive(Debug)]
+#[derive(Clone)]
+pub struct ObjectRef {
+    index: usize,
+}
+
+#[derive(Debug)]
+pub struct ObjectStore {
+    all_objects: Vec<Object>,
+}
+
+impl ObjectStore {
+    pub fn new() -> ObjectStore {
+        ObjectStore { all_objects: Vec::new() }
+    }
+
+    pub fn allocate(&mut self, obj: ObjectContent) -> ObjectRef {
+        let obj_ref = ObjectRef { index: self.all_objects.len() };
+        self.all_objects.push(Object { content: obj });
+        obj_ref
+    }
+}