Update to new task (#33)

- Import code from jupyter notebook
- Adding iotlabcli python lib
- Migrate to Python3.4
- Install all apt packages in one go
- Moving to fabric3 for Python3 support
- Correct the print
- Pinpoint the java version used
- Adding a Makefile with connect command
- Adding a tunslip to the CSC
- Adding new cooja launch
- Adding a cooja_launch
- Adding coap traffic
- Adding pycrypto

.travis.yml

@@ -1,13 +1,19 @@
 language: python
+sudo: required
+dist: trusty
+
+python:
+  - "3.4"
 
 before_install:
     # Managing python dependencies through pip is the way to go. But it can be very long
     # to compile every time all the dependencies therefore to speedup unit testing we will
     # use binary packages
   - "sudo apt-get update"
-  - "sudo apt-get -qq install ipython python-matplotlib fabric python-pandas python-networkx python-numpy python-jinja2 python-scipy"
+  - "sudo apt-get -qq install ipython ipython3 python3-matplotlib python-matplotlib python3-pandas python-pandas python-networkx python3-numpy python-numpy python-jinja2 python-scipy python3-scipy tshark"
     #- "pip install -r requirements.txt --use-mirrors"
 
+    # While https://github.com/contiki-os/contiki/pull/1557 is not merged, we use our own Contiki repository
   - "git clone --recursive git://github.com/contiki-os/contiki"
   - WGET="travis_retry wget --continue --tries=20 --waitretry=10 --retry-connrefused --no-dns-cache --timeout 300"
   - sudo apt-get -qq update
@@ -18,15 +24,18 @@ before_install:
     rm -rf /tmp/msp430 mspgcc*.tar.bz2 &&
     msp430-gcc --version
 
-    # Useful to analyze PCAP traces
-  - "sudo apt-get install tshark"
-
     # Install a ruby coap client
   - "gem install coap"
 
+    # Contiki seems to work only with oracle jdk1.7
+  - jdk_switcher use oraclejdk7
+
 script:
   - "ipython nbconvert --to=python demo.ipynb"
   - python demo.py
+  - fab new:test
+  - fab make:test
+  - fab launch_cooja:test
 
 cache: pip
 

demo.ipynb

@@ -483,7 +483,7 @@
     "            f.write(json.dumps({\"id\": testbed_experiment_id}))\n",
     "\n",
     "    # get the content\n",
-    "    print \"Exp submited with id: %u\" % testbed_experiment_id\n",
+    "    print(\"Exp submited with id: %u\" % testbed_experiment_id)\n",
     "    \n",
     "    # We wait till the experiment is completed\n",
     "    # note that this step requieres you to have experiment-cli installed on the command line.\n",
@@ -841,7 +841,7 @@
     "    %matplotlib inline\n",
     "    \n",
     "    for host in testbed_all_nodes:\n",
-    "        print testbed_df[host].power.plot()"
+    "        testbed_df[host].power.plot()"
    ]
   },
   {
@@ -879,7 +879,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3",
+   "display_name": "IPython (Python 3)",
    "language": "python",
    "name": "python3"
   },

fabfile.py

@@ -48,10 +48,19 @@
 TEMPLATE_FOLDER = pj(ROOT_DIR, "templates")
 TEMPLATE_ENV = Environment(loader=FileSystemLoader(TEMPLATE_FOLDER))
 COOJA_DIR = pj(CONTIKI_FOLDER, "tools", "cooja")
+TUNSLIP6 = pj(CONTIKI_FOLDER, "tools", "tunslip6")
 
+settings = {
+        "ROOT_DIR": ROOT_DIR,
+        "CONTIKI_FOLDER": CONTIKI_FOLDER,
+        "EXPERIMENT_FOLDER": EXPERIMENT_FOLDER,
+        "TEMPLATE_FOLDER": TEMPLATE_FOLDER,
+        "COOJA_DIR": COOJA_DIR,
+        "TUNSLIP6": TUNSLIP6
+}
 
 @task
-def new(name):
+def new(name, notebook_template="Default Jupyter notebook.ipynb"):
     """
     Default experiment is a client and a server sending messages to each others.
     The default platform is wismote
@@ -60,48 +69,16 @@ def new(name):
     if not os.path.exists(path):
         os.makedirs(path)
 
-    client_template = TEMPLATE_ENV.get_template("dummy_client.c")
-    with open(pj(path, "dummy-client.c"), "w") as f:
-        f.write(client_template.render())
-
-    server_template = TEMPLATE_ENV.get_template("dummy_server.c")
-    with open(pj(path, "dummy-server.c"), "w") as f:
-        f.write(server_template.render())
-
-    makefile_template = TEMPLATE_ENV.get_template("dummy_makefile")
-    with open(pj(path, "Makefile"), "w") as f:
-        f.write(makefile_template.render(contiki=CONTIKI_FOLDER,
-                                         target="wismote"))
-
-    main_csc_template = TEMPLATE_ENV.get_template("dummy_main.csc")
-    script_template = TEMPLATE_ENV.get_template("dummy_script.js")
-    script = script_template.render(
-        timeout=100000,
-        powertracker_frequency=10000,
-    )
-    with open(pj(path, "main.csc"), "w") as f:
-        f.write(main_csc_template.render(
-            title="Dummy Simulation",
-            random_seed=12345,
-            transmitting_range=42,
-            interference_range=42,
-            success_ratio_tx=1.0,
-            success_ratio_rx=1.0,
-            mote_types=[
-                {"name": "server", "description": "server",
-                    "firmware": "dummy-server.wismote"},
-                {"name": "client", "description": "client",
-                    "firmware": "dummy-client.wismote"}
-            ],
-            motes=[
-                {"mote_id": 1, "x": 0, "y": 0, "z": 0, "mote_type": "server"},
-                {"mote_id": 2, "x": 1, "y": 1, "z": 0, "mote_type": "client"},
-            ],
-            script=script))
-
-    print(
-        "Think to rename otherwise if you do fab new:%s again dummy files will be overwritten" % name)
+    # Copy of the settings file
+    with open(pj(path, "settings.json"), "w") as f:
+        f.write(json.dumps(settings,
+                sort_keys=True,
+                indent=4,
+                separators=(',', ': ')))
 
+    notebook_template = TEMPLATE_ENV.get_template(notebook_template)
+    with open(pj(path, "%s.ipynb" % name), "w") as f:
+        f.write(notebook_template.render())
 
 @task
 def new_iotlab(name):
@@ -133,18 +110,6 @@ def new_iotlab(name):
     with open(pj(path, "iotlab.json"), "w") as f:
         f.write(config_template.render())
 
-
-@task
-def compile_nodes(name, target="wismote"):
-    """
-    Fabric command to compile nodes
-    """
-    path = pj(EXPERIMENT_FOLDER, name)
-    with lcd(path):
-        local("make TARGET=%s" % target)
-        local("make ihex")
-
-
 @task
 @hosts("grenoble")
 def push_iotlab(name):
@@ -309,16 +274,6 @@ def clean_nodes(name):
     with lcd(path):
         local("make clean")
 
-
-@task
-def make(name):
-    """
-    Fabric command to make all the folder structures and compile nodes
-    """
-    compile_nodes(name)
-    compile_cooja()
-
-
 @task
 def launch(name):
     """
@@ -328,7 +283,6 @@ def launch(name):
     with lcd(path):
         local("make run")
 
-
 @task
 def clean_results(name):
     """
@@ -376,14 +330,6 @@ def pull(name):
     path = pj(EXPERIMENT_FOLDER, name)
     get(name, local_path=path)
 
-
-@task
-@hosts("galois")
-def pull_results(name):
-    path = pj(EXPERIMENT_FOLDER, name)
-    get(pj(name, "results"), local_path=path)
-
-
 @task
 def plot(name):
     path = pj(EXPERIMENT_FOLDER, name)
@@ -399,17 +345,6 @@ def plot(name):
     pt.energy(path)
     pt.energy_depth(path)
 
-
-@task
-def notebook(name):
-    notebook_template = TEMPLATE_ENV.get_template("dummy.ipynb")
-    path = pj(EXPERIMENT_FOLDER, name)
-    with open(pj(path, "%s.ipynb" % name), "w") as f:
-        f.write(notebook_template.render())
-    print("Run with fab web:%(name)s (ipython notebook %(path)s/%(name)s.ipynb)" %
-          {"path": path, "name": name})
-
-
 @task
 @parallel
 @roles("iotlab")
@@ -436,90 +371,43 @@ def web(name):
     Fabric command to launch a web server with ipython
     """
     path = pj(EXPERIMENT_FOLDER, name)
-    local("ipython notebook %(path)s/%(name)s.ipynb" %
+    local("jupyter-notebook %(path)s/%(name)s.ipynb" %
           {"path": path, "name": name})
 
-import io, os, sys, types
 import nbformat
-from IPython.core.interactiveshell import InteractiveShell
+from nbconvert import PythonExporter
+
+def update_script(name):
+    python_exporter = PythonExporter()
+    with open("experiments/{name}/{name}.ipynb".format(name=name)) as f:
+        notebook = nbformat.reads(f.read(), 4)
+        test = python_exporter.from_notebook_node(notebook)
+        path = "experiments/{name}/{name}.py".format(name=name)
+        with open(path, "w") as f_out:
+            f_out.write(test[0])
+        return path
 
-def find_notebook(fullname, path=None):
-    """find a notebook, given its fully qualified name and an optional path
+@task
+def make(name):
+    """
+    Fabric command to make all the folder structures and compile nodes
+    """
+    # In python 3.5
+    #import importlib.util
+    #spec = importlib.util.spec_from_file_location("module.name", "/path/to/file.py")
+    #foo = importlib.util.module_from_spec(spec)
+    #spec.loader.exec_module(foo)
+    #foo.MyClass()
+
+    path = update_script(name)
+    from importlib.machinery import SourceFileLoader
+    SourceFileLoader(name, path).load_module().make()
 
-    This turns "foo.bar" into "foo/bar.ipynb"
-    and tries turning "Foo_Bar" into "Foo Bar" if Foo_Bar
-    does not exist.
+@task
+def launch_cooja(name):
+    """
+    Fabric command to launch an experiment
     """
-    name = fullname.rsplit('.', 1)[-1]
-    if not path:
-        path = ['']
-    for d in path:
-        nb_path = os.path.join(d, name + ".ipynb")
-        if os.path.isfile(nb_path):
-            return nb_path
-        # let import Notebook_Name find "Notebook Name.ipynb"
-        nb_path = nb_path.replace("_", " ")
-        if os.path.isfile(nb_path):
-            return nb_path
-
-class NotebookLoader(object):
-    """Module Loader for IPython Notebooks"""
-    def __init__(self, path=None):
-        self.shell = InteractiveShell.instance()
-        self.path = path
-
-    def load_module(self, fullname):
-        """import a notebook as a module"""
-        path = find_notebook(fullname, self.path)
-
-        print ("importing IPython notebook from %s" % path)
-
-        # load the notebook object
-        with open(path, 'r', encoding='utf-8') as f:
-            nb = nbformat.read(f, 3)
-
-
-        # create the module and add it to sys.modules
-        # if name in sys.modules:
-        #    return sys.modules[name]
-        mod = types.ModuleType(fullname)
-        mod.__file__ = path
-        mod.__loader__ = self
-        sys.modules[fullname] = mod
-
-        # extra work to ensure that magics that would affect the user_ns
-        # actually affect the notebook module's ns
-        save_user_ns = self.shell.user_ns
-        self.shell.user_ns = mod.__dict__
-
-        try:
-          for cell in nb.worksheets[0].cells:
-            if cell.cell_type == 'code' and cell.language == 'python':
-                # transform the input to executable Python
-                code = self.shell.input_transformer_manager.transform_cell(cell.input)
-                # run the code in themodule
-                exec(code, mod.__dict__)
-        finally:
-            self.shell.user_ns = save_user_ns
-        return mod
-
-class NotebookFinder(object):
-    """Module finder that locates IPython Notebooks"""
-    def __init__(self):
-        self.loaders = {}
-
-    def find_module(self, fullname, path=None):
-        nb_path = find_notebook(fullname, path)
-        if not nb_path:
-            return
-
-        key = path
-        if path:
-            # lists aren't hashable
-            key = os.path.sep.join(path)
-
-        if key not in self.loaders:
-            self.loaders[key] = NotebookLoader(path)
-        return self.loaders[key]
-
-sys.meta_path.append(NotebookFinder())
+    path = update_script(name)
+    from importlib.machinery import SourceFileLoader
+    SourceFileLoader(name, path).load_module().launch_cooja()

makesense/parser.py

@@ -235,7 +235,7 @@ def frame_size(size):
     d = {'udp': 8, 'ipv6_hop': 8, '6lowpan': 6, '802.15.4': 23}
     if size > 50:
         log.warning('Fragmentation occurred I cannot compute reliably')
-    total = sum(d.values() + [size])
+    total = sum(list(d.values()) + [size])
     log.debug("frame_size Payload size: %d ; On wire size %d", size, total)
     return total
 

requirements.txt

@@ -1,9 +1,10 @@
 ipython[all]==4.1.1
 Jinja2==2.8
-Fabric==1.10.2
+Fabric3==1.10.2.post3
 networkx==1.11
 requests==2.9.1
 pandas==0.17.1
+pycrypto==2.6.1
 
 # Scientific libraries take a lot of time to build.
 #scipy==0.14