Merge pull request #44 from GuillaumeDerval/yaml

.gitignore

@@ -4,9 +4,9 @@
 *.py[cod]
 sessions*
 repo_submissions
-configuration.json
-configuration.*.json
-!configuration.example.json
+configuration.yaml
+configuration.*.yaml
+!configuration.example.yaml
 dump
 doc/_*
 tasks

app_frontend.py

@@ -19,6 +19,8 @@
 # License along with INGInious.  If not, see <http://www.gnu.org/licenses/>.
 """ Starts the frontend """
 
+import os.path
+
 import web
 
 import common.base
@@ -76,5 +78,10 @@ def not_found():
     return appli
 
 if __name__ == "__main__":
-    app = get_app("./configuration.json")
+    if os.path.isfile("./configuration.yaml"):
+        app = get_app("./configuration.yaml")
+    elif os.path.isfile("./configuration.json"):
+        app = get_app("./configuration.json")
+    else:
+        raise Exception("No configuration file found")
     app.run()

common/base.py

@@ -17,21 +17,44 @@
 # You should have received a copy of the GNU Affero General Public
 # License along with INGInious.  If not, see <http://www.gnu.org/licenses/>.
 """ Basic dependencies for every modules that uses INGInious """
+import codecs
 import json
+import os.path
 import re
 
+import common.custom_yaml
+
 
 class Configuration(dict):
 
     """ Config class """
 
     def load(self, path):
-        """ Load the config from a json file """
-        self.update(json.load(open(path, "r")))
+        """ Load the config from a file """
+        self.update(load_json_or_yaml(path))
 
 INGIniousConfiguration = Configuration()
 
 
 def id_checker(id_to_test):
     """Checks if a id is correct"""
     return bool(re.match(r'[a-z0-9\-_]+$', id_to_test, re.IGNORECASE))
+
+
+def load_json_or_yaml(file_path):
+    """ Load JSON or YAML depending on the file extension. Returns a dict """
+    if os.path.splitext(file_path)[1] == ".json":
+        return json.load(open(file_path, "r"))
+    else:
+        return common.custom_yaml.load(open(file_path, "r"))
+
+
+def write_json_or_yaml(file_path, content):
+    """ Load JSON or YAML depending on the file extension. """
+    if os.path.splitext(file_path)[1] == ".json":
+        o = json.dumps(content, sort_keys=False, indent=4, separators=(',', ': '))
+    else:
+        o = common.custom_yaml.dump(content)
+
+    with codecs.open(file_path, "w", "utf-8") as f:
+        f.write(o)

common/courses.py

@@ -17,12 +17,10 @@
 # You should have received a copy of the GNU Affero General Public
 # License along with INGInious.  If not, see <http://www.gnu.org/licenses/>.
 """ Contains the class Course and utility functions """
-import json
-import os
 import os.path
 
-from common.base import INGIniousConfiguration, id_checker
-from common.task_file_managers.tasks_file_manager import TaskFileManager
+from common.base import INGIniousConfiguration, id_checker, load_json_or_yaml, write_json_or_yaml
+from common.task_file_managers.manage import get_readable_tasks
 import common.tasks
 
 
@@ -33,16 +31,33 @@ class Course(object):
     _task_class = common.tasks.Task
 
     @classmethod
-    def get_course_descriptor_path(cls, courseid):
-        """Returns the path to the json that describes the course 'courseid'"""
+    def _get_course_descriptor_path(cls, courseid):
+        """Returns the path to the file that describes the course 'courseid'"""
         if not id_checker(courseid):
             raise Exception("Course with invalid name: " + courseid)
-        return os.path.join(INGIniousConfiguration["tasks_directory"], courseid, "course.json")
+        base_file = os.path.join(INGIniousConfiguration["tasks_directory"], courseid, "course")
+        if os.path.isfile(base_file + ".yaml"):
+            return base_file + ".yaml"
+        else:
+            return base_file + ".json"
+        return base_file + ".yaml"  # by default, YAML.
+
+    @classmethod
+    def get_course_descriptor_content(cls, courseid):
+        """ Returns the content of the dict that describes the course """
+        return load_json_or_yaml(cls._get_course_descriptor_path(courseid))
+
+    @classmethod
+    def update_course_descriptor_content(cls, courseid, content):
+        """ Updates the content of the dict that describes the course """
+        return write_json_or_yaml(cls._get_course_descriptor_path(courseid), content)
 
     @classmethod
     def get_all_courses(cls):
         """Returns a table containing courseid=>Course pairs."""
-        files = [os.path.splitext(f)[0] for f in os.listdir(INGIniousConfiguration["tasks_directory"]) if os.path.isfile(os.path.join(INGIniousConfiguration["tasks_directory"], f, "course.json"))]
+        files = [os.path.splitext(f)[0] for f in os.listdir(INGIniousConfiguration["tasks_directory"]) if
+                 os.path.isfile(os.path.join(INGIniousConfiguration["tasks_directory"], f, "course.yaml")) or
+                 os.path.isfile(os.path.join(INGIniousConfiguration["tasks_directory"], f, "course.json"))]
         output = {}
         for course in files:
             try:
@@ -53,15 +68,10 @@ def get_all_courses(cls):
 
     def __init__(self, courseid):
         """Constructor. courseid is the name of the the folder containing the file course.json"""
-
-        self._content = json.load(open(self.get_course_descriptor_path(courseid), "r"))
+        self._content = self.get_course_descriptor_content(courseid)
         self._id = courseid
         self._tasks_cache = None
 
-    def get_original_content(self):
-        """ Return the original content of the json file describing this course """
-        return self._content
-
     def get_task(self, taskid):
         """ Return the class with name taskid """
         return self._task_class(self, taskid)
@@ -77,7 +87,7 @@ def get_course_tasks_directory(self):
     def get_tasks(self):
         """Get all tasks in this course"""
         if self._tasks_cache is None:
-            tasks = TaskFileManager.get_tasks(self.get_id())
+            tasks = get_readable_tasks(self.get_id())
             output = {}
             for task in tasks:
                 try:

common/custom_yaml.py

@@ -0,0 +1,91 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2014-2015 Université Catholique de Louvain.
+#
+# This file is part of INGInious.
+#
+# INGInious is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# INGInious is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public
+# License along with INGInious.  If not, see <http://www.gnu.org/licenses/>.
+""" A custom YAML based on PyYAML, that provides Ordered Dicts """
+# Most ideas for this implementation comes from http://stackoverflow.com/questions/5121931/in-python-how-can-you-load-yaml-mappings-as-ordereddicts
+from collections import OrderedDict
+
+import yaml as original_yaml
+
+
+def load(stream):
+    """
+        Parse the first YAML document in a stream
+        and produce the corresponding Python
+        object. Use OrderedDicts to produce dicts.
+
+        Safe version.
+    """
+    class OrderedLoader(original_yaml.SafeLoader):
+        pass
+
+    def construct_mapping(loader, node):
+        loader.flatten_mapping(node)
+        return OrderedDict(loader.construct_pairs(node))
+
+    OrderedLoader.add_constructor(
+        original_yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
+        construct_mapping)
+
+    return original_yaml.load(stream, OrderedLoader)
+
+
+def dump(data, stream=None, **kwds):
+    """
+        Serialize a Python object into a YAML stream.
+        If stream is None, return the produced string instead.
+        Dict keys are produced in the order in which they appear in OrderedDicts.
+
+        Safe version.
+
+        If objects are not "conventional" objects, they will be dumped converted to string with the str() function.
+        They will then not be recovered when loading with the load() function.
+    """
+
+    # Display OrderedDicts correctly
+    class OrderedDumper(original_yaml.SafeDumper):
+        pass
+
+    def _dict_representer(dumper, data):
+        return dumper.represent_mapping(
+            original_yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
+            data.items())
+
+    # Display long strings correctly
+    def _long_str_representer(dumper, data):
+        if data.find("\n") != -1:
+            # Drop some uneeded data
+            # \t are forbidden in YAML
+            data = data.replace("\t", "    ")
+            # empty spaces at end of line are always useless in INGInious, and forbidden in YAML
+            data = "\n".join([p.rstrip() for p in data.split("\n")])
+            return dumper.represent_scalar(u'tag:yaml.org,2002:str', data, style='|')
+        else:
+            return dumper.represent_scalar(u'tag:yaml.org,2002:str', data)
+
+    # Default representation for some odd objects
+    def _default_representer(dumper, data):
+        return _long_str_representer(dumper, str(data))
+
+    OrderedDumper.add_representer(str, _long_str_representer)
+    OrderedDumper.add_representer(unicode, _long_str_representer)
+    OrderedDumper.add_representer(OrderedDict, _dict_representer)
+    OrderedDumper.add_representer(None, _default_representer)
+
+    s = original_yaml.dump(data, stream, OrderedDumper, encoding='utf-8', allow_unicode=True, default_flow_style=False, indent=4, **kwds)
+    return s.decode('utf-8')

common/task_file_managers/__init__.py

@@ -1,3 +0,0 @@
-""" Classes to parse and write task's descriptions """
-import common.task_file_managers.tasks_json_file_manager
-import common.task_file_managers.tasks_rst_file_manager

common/task_file_managers/tasks_file_manager.py → common/task_file_managers/abstract_manager.py

@@ -24,7 +24,7 @@
 from common.base import INGIniousConfiguration
 
 
-class TaskFileManager(object):
+class AbstractTaskFileManager(object):
 
     """ Manages a type of task file """
     __metaclass__ = ABCMeta
@@ -56,50 +56,3 @@ def write(self, data):
     def _generate_content(self, data):
         """ Generate data (that will be written to the file) """
         pass
-
-    @classmethod
-    def get_tasks(cls, courseid):
-        """ Returns the list of all available tasks in a course """
-        tasks = [
-            task for task in os.listdir(
-                os.path.join(
-                    INGIniousConfiguration["tasks_directory"],
-                    courseid)) if os.path.isdir(os.path.join(
-                        INGIniousConfiguration["tasks_directory"],
-                        courseid,
-                        task)) and cls._task_file_exists(
-                os.path.join(
-                    INGIniousConfiguration["tasks_directory"],
-                    courseid,
-                    task))]
-        return tasks
-
-    @classmethod
-    def _task_file_exists(cls, directory):
-        """ Returns true if a task file exists in this directory """
-        for filename in ["task.{}".format(subclass.get_ext()) for subclass in TaskFileManager.__subclasses__()]:
-            if os.path.isfile(os.path.join(directory, filename)):
-                return True
-        return False
-
-    @classmethod
-    def get_manager(cls, courseid, taskid):
-        """ Returns the appropriate task file manager for this task """
-        for subclass in TaskFileManager.__subclasses__():
-            if os.path.isfile(os.path.join(INGIniousConfiguration["tasks_directory"], courseid, taskid, "task.{}".format(subclass.get_ext()))):
-                return subclass(courseid, taskid)
-        return None
-
-    @classmethod
-    def delete_all_possible_task_files(cls, courseid, taskid):
-        """ Deletes all possibles task files in directory, to allow to change the format """
-        for subclass in TaskFileManager.__subclasses__():
-            try:
-                os.remove(os.path.join(INGIniousConfiguration["tasks_directory"], courseid, taskid, "task.{}".format(subclass.get_ext())))
-            except:
-                pass
-
-    @classmethod
-    def get_available_file_managers(cls):
-        """ Get a dict with ext:class pairs """
-        return {f.get_ext(): f for f in TaskFileManager.__subclasses__()}

common/task_file_managers/manage.py

@@ -0,0 +1,50 @@
+import os.path
+
+from common.base import INGIniousConfiguration
+from common.task_file_managers.yaml_manager import TaskYAMLFileManager
+
+_task_file_managers = [TaskYAMLFileManager]
+
+
+def get_readable_tasks(courseid):
+    """ Returns the list of all available tasks in a course """
+    tasks = [
+        task for task in os.listdir(os.path.join(INGIniousConfiguration["tasks_directory"], courseid))
+        if os.path.isdir(os.path.join(INGIniousConfiguration["tasks_directory"], courseid, task))
+        and _task_file_exists(os.path.join(INGIniousConfiguration["tasks_directory"], courseid, task))]
+    return tasks
+
+
+def _task_file_exists(directory):
+    """ Returns true if a task file exists in this directory """
+    for filename in ["task.{}".format(ext) for ext in get_available_task_file_managers().keys()]:
+        if os.path.isfile(os.path.join(directory, filename)):
+            return True
+    return False
+
+
+def get_task_file_manager(courseid, taskid):
+    """ Returns the appropriate task file manager for this task """
+    for ext, subclass in get_available_task_file_managers().iteritems():
+        if os.path.isfile(os.path.join(INGIniousConfiguration["tasks_directory"], courseid, taskid, "task.{}".format(ext))):
+            return subclass(courseid, taskid)
+    return None
+
+
+def delete_all_possible_task_files(courseid, taskid):
+    """ Deletes all possibles task files in directory, to allow to change the format """
+    for ext in get_available_task_file_managers().keys():
+        try:
+            os.remove(os.path.join(INGIniousConfiguration["tasks_directory"], courseid, taskid, "task.{}".format(ext)))
+        except:
+            pass
+
+
+def add_custom_task_file_manager(task_file_manager):
+    """ Add a custom task file manager """
+    _task_file_managers.append(task_file_manager)
+
+
+def get_available_task_file_managers():
+    """ Get a dict with ext:class pairs """
+    return {f.get_ext(): f for f in _task_file_managers}

common/task_file_managers/tasks_rst_file_manager.py → common/task_file_managers/yaml_manager.py

@@ -16,23 +16,22 @@
 #
 # You should have received a copy of the GNU Affero General Public
 # License along with INGInious.  If not, see <http://www.gnu.org/licenses/>.
-""" RST task file manager """
+""" YAML task file manager """
 
-from common.task_file_managers._dicttorst import dict2rst
-from common.task_file_managers._rsttodict import rst2dict
-from common.task_file_managers.tasks_file_manager import TaskFileManager
+import common.custom_yaml
+from common.task_file_managers.abstract_manager import AbstractTaskFileManager
 
 
-class TaskRSTFileManager(TaskFileManager):
+class TaskYAMLFileManager(AbstractTaskFileManager):
 
-    """ Read and write task descriptions in restructuredText """
+    """ Read and write task descriptions in YAML """
 
     def _get_content(self, content):
-        return rst2dict(content)
+        return common.custom_yaml.load(content)
 
     @classmethod
     def get_ext(cls):
-        return "rst"
+        return "yaml"
 
     def _generate_content(self, data):
-        return dict2rst(data)
+        return common.custom_yaml.dump(data)