Port to python 3

hotdoc/core/base_extension.py

@@ -19,7 +19,6 @@
 """
 Utilities and baseclasses for extensions
 """
-
 import os
 from collections import defaultdict
 
@@ -101,7 +100,7 @@ def __init__(self, doc_repo):
         if not hasattr(self, 'formatters'):
             self.formatters = {"html": HtmlFormatter([])}
 
-        self.__created_symbols = defaultdict(OrderedSet)
+        self._created_symbols = defaultdict(OrderedSet)
         self.__package_root = None
 
     # pylint: disable=no-self-use
@@ -168,7 +167,7 @@ def reset(self):
         This function is only useful for testing purposes, at least
         for now.
         """
-        self.__created_symbols = defaultdict(OrderedSet)
+        self._created_symbols = defaultdict(OrderedSet)
         self.__package_root = None
 
     def setup(self):
@@ -224,11 +223,11 @@ def parse_standard_config(cls, config):
         cls.smart_index = bool(config.get('%s_smart_index' %
                                           cls.argument_prefix))
 
-        for arg, dest in cls.paths_arguments.items():
+        for arg, dest in list(cls.paths_arguments.items()):
             val = config.get_paths(arg)
             setattr(cls, dest, val)
 
-        for arg, dest in cls.path_arguments.items():
+        for arg, dest in list(cls.path_arguments.items()):
             val = config.get_path(arg)
             setattr(cls, dest, val)
 
@@ -349,11 +348,9 @@ def get_or_create_symbol(self, *args, **kwargs):
             symbols.Symbol: the created symbol, or `None`.
         """
         sym = self.doc_repo.doc_database.get_or_create_symbol(*args, **kwargs)
-
         # pylint: disable=unidiomatic-typecheck
         if sym and type(sym) != Symbol and sym.filename:
-            # assert sym.filename is not None
-            self.__created_symbols[sym.filename].add(sym.unique_name)
+            self._created_symbols[sym.filename].add(sym.unique_name)
 
         return sym
 
@@ -387,12 +384,12 @@ def __update_doc_tree_cb(self, doc_tree, unlisted_sym_names):
         for sym_name in unlisted_sym_names:
             sym = self.doc_repo.doc_database.get_symbol(sym_name)
             if sym and sym.filename in self._get_all_sources():
-                self.__created_symbols[sym.filename].add(sym_name)
+                self._created_symbols[sym.filename].add(sym_name)
 
         user_pages = [p for p in doc_tree.walk(index) if not p.generated]
         user_symbols = self.__get_user_symbols(user_pages)
 
-        for source_file, symbols in self.__created_symbols.items():
+        for source_file, symbols in list(self._created_symbols.items()):
             gen_symbols = symbols - user_symbols
             self.__add_subpage(doc_tree, index, source_file, gen_symbols)
             doc_tree.stale_symbol_pages(symbols)

hotdoc/core/base_formatter.py

@@ -19,24 +19,24 @@
 """
 This module defines a base Formatter class
 """
-
 import os
 import shutil
 
 from collections import defaultdict
 
 from schema import Schema, Optional
+import pygraphviz as pg
+
 from hotdoc.core.doc_tree import Page
 from hotdoc.utils.utils import symlink
 
-import pygraphviz as pg
 from hotdoc.utils.configurable import Configurable
 from hotdoc.utils.simple_signals import Signal
 from hotdoc.utils.utils import recursive_overwrite, OrderedSet
 
 
 Page.meta_schema[Optional('extra', default=defaultdict())] = \
-    Schema({unicode: object})
+    Schema({str: object})
 
 
 def _create_hierarchy_graph(hierarchy):
@@ -166,8 +166,7 @@ def write_page(self, page, root, output):
         self.writing_page_signal(self, page, path)
 
         with open(path, 'w') as _:
-            out = page.detailed_description
-            _.write(out.encode('utf-8'))
+            _.write(page.detailed_description)
 
         self.__copy_extra_files(root, os.path.dirname(path))
 

hotdoc/core/change_tracker.py

@@ -19,7 +19,6 @@
 """
 Defines and tests ChangeTracker
 """
-
 import os
 from collections import defaultdict
 
@@ -82,7 +81,7 @@ def hard_dependencies_are_stale(self):
         """
         Banana banana
         """
-        for filename, last_mtime in self.hard_deps_mtimes.items():
+        for filename, last_mtime in list(self.hard_deps_mtimes.items()):
             mtime = get_mtime(filename)
 
             if mtime == -1 or mtime != last_mtime:
@@ -95,31 +94,31 @@ def hard_dependencies_are_stale(self):
 
     # Initial build
     os.system('touch a b c d')
-    print "Should be ([a, b, c, d], [])"
-    print CTRACKER.get_stale_files(['a', 'b', 'c', 'd'], 'testing')
+    print("Should be ([a, b, c, d], [])")
+    print(CTRACKER.get_stale_files(['a', 'b', 'c', 'd'], 'testing'))
 
     # Build where nothing changed
-    print "Should be ([], [])"
-    print CTRACKER.get_stale_files(['a', 'b', 'c', 'd'], 'testing')
+    print("Should be ([], [])")
+    print(CTRACKER.get_stale_files(['a', 'b', 'c', 'd'], 'testing'))
 
     # Build with two files changed
     os.system('touch b d')
-    print "Should be ([b, d], [])"
-    print CTRACKER.get_stale_files(['a', 'b', 'c', 'd'], 'testing')
+    print("Should be ([b, d], [])")
+    print(CTRACKER.get_stale_files(['a', 'b', 'c', 'd'], 'testing'))
 
     # Build where one file was removed
     os.system('rm -f b')
-    print "Should be ([b], [])"
-    print CTRACKER.get_stale_files(['a', 'b', 'c', 'd'], 'testing')
-    print "Should be ([], [])"
-    print CTRACKER.get_stale_files(['a', 'b', 'c', 'd'], 'testing')
+    print("Should be ([b], [])")
+    print(CTRACKER.get_stale_files(['a', 'b', 'c', 'd'], 'testing'))
+    print("Should be ([], [])")
+    print(CTRACKER.get_stale_files(['a', 'b', 'c', 'd'], 'testing'))
 
     # Build where one file was unlisted
-    print "Should be ([], [a])"
-    print CTRACKER.get_stale_files(['b', 'c', 'd'], 'testing')
+    print("Should be ([], [a])")
+    print(CTRACKER.get_stale_files(['b', 'c', 'd'], 'testing'))
 
     # Build with file listed again
-    print "Should be ([a], [])"
-    print CTRACKER.get_stale_files(['a', 'b', 'c', 'd'], 'testing')
+    print("Should be ([a], [])")
+    print(CTRACKER.get_stale_files(['a', 'b', 'c', 'd'], 'testing'))
 
     os.system('rm -f a b c d')

hotdoc/core/comment_block.py

@@ -20,7 +20,6 @@
 This module implements a Comment class, to be used
 by code-parsing extensions.
 """
-
 import linecache
 import os
 from collections import defaultdict
@@ -69,7 +68,7 @@ def __init__(self, name=u'', title=None, params=None, filename=u'',
         self.col_offset = 0
         self.initial_col_offset = 0
         self.annotations = annotations or {}
-        self.description = unicode(description)
+        self.description = str(description)
         self.short_description = short_description
         self.extension_attrs = defaultdict(lambda: defaultdict(dict))
         self.tags = tags or {}

hotdoc/core/config.py

@@ -19,7 +19,6 @@
 """
 Implement a high-level config parser for hotdoc.
 """
-
 import os
 import json
 import glob
@@ -308,14 +307,14 @@ def get_dependencies(self):
                 tracked configuration.
         """
         all_deps = OrderedSet()
-        for key, _ in self.__config.items():
+        for key, _ in list(self.__config.items()):
             if key in self.__cli:
                 continue
 
             if key.endswith('sources'):
                 all_deps |= self.get_sources(key[:len('sources') * -1])
 
-        for key, _ in self.__cli.items():
+        for key, _ in list(self.__cli.items()):
             if key.endswith('sources'):
                 all_deps |= self.get_sources(key[:len('sources') * -1])
 
@@ -344,12 +343,12 @@ def dump(self, conf_file=None):
             conf_dir = self.__conf_dir
 
         final_conf = {}
-        for key, value in self.__config.items():
+        for key, value in list(self.__config.items()):
             if key in self.__cli:
                 continue
             final_conf[key] = value
 
-        for key, value in self.__cli.items():
+        for key, value in list(self.__cli.items()):
             if (key.endswith('index') and not key.endswith('smart_index')) or \
                     key in ['sitemap', 'output']:
                 path = self.__abspath(value, from_conf=False)

hotdoc/core/doc_database.py

@@ -18,7 +18,6 @@
 
 """Banana banana
 """
-
 import os
 
 # pylint: disable=import-error
@@ -91,7 +90,7 @@ def get_or_create_symbol(self, type_, **kwargs):
             debug('Created symbol with unique name %s' % unique_name,
                   'symbols')
 
-        for key, value in kwargs.items():
+        for key, value in list(kwargs.items()):
             setattr(symbol, key, value)
 
         if self.__incremental:

hotdoc/core/doc_repo.py

@@ -19,15 +19,16 @@
 """
 Core of the core.
 """
-
 import argparse
 import hashlib
-import cPickle as pickle
+import pickle as pickle
 import os
 import shutil
 import sys
 import io
 
+from collections import OrderedDict
+
 from hotdoc.core import file_includer
 from hotdoc.core.base_extension import BaseExtension
 from hotdoc.core.change_tracker import ChangeTracker
@@ -117,8 +118,8 @@ def __init__(self):
             self.datadir = "/usr/share"
 
         self.__conf_file = None
-        self.__extension_classes = {
-            CoreExtension.extension_name: CoreExtension}
+        self.__extension_classes = OrderedDict({
+            CoreExtension.extension_name: CoreExtension})
         self.__index_file = None
         self.__root_page = None
         self.__base_doc_folder = None
@@ -149,8 +150,8 @@ def patch_page(self, symbol, raw_comment):
         """
         Banana banana
         """
-        pages = self.doc_tree.get_pages_for_symbol(symbol.unique_name)
-        if not pages:
+        page = self.doc_tree.get_page_for_symbol(symbol.unique_name)
+        if not page:
             return False
 
         old_comment = symbol.comment
@@ -168,11 +169,9 @@ def patch_page(self, symbol, raw_comment):
         if new_comment.name != symbol.unique_name:
             return False
 
-        pages = pages.values()
         symbol.comment = new_comment
-        for page in pages:
-            formatter = self.__get_formatter(page.extension_name)
-            formatter.patch_page(page, symbol, self.output)
+        formatter = self.__get_formatter(page.extension_name)
+        formatter.patch_page(page, symbol, self.output)
 
         return True
 
@@ -224,7 +223,7 @@ def setup(self):
 
         self.doc_tree = DocTree(self.get_private_folder(), self.include_paths)
 
-        for extension in self.extensions.values():
+        for extension in list(self.extensions.values()):
             info('Setting up %s' % extension.extension_name)
             extension.setup()
             self.doc_database.flush()
@@ -271,7 +270,7 @@ def __dump_deps_file(self):
                     _.write(u'%s ' % dep)
 
             if self.doc_tree:
-                for page in self.doc_tree.get_pages().values():
+                for page in list(self.doc_tree.get_pages().values()):
                     if not page.generated:
                         empty_targets.append(page.source_file)
                         _.write(u'%s ' % page.source_file)
@@ -280,7 +279,7 @@ def __dump_deps_file(self):
                 _.write(u'\n\n%s:' % empty_target)
 
     def __add_default_tags(self, _, comment):
-        for validator in self.tag_validators.values():
+        for validator in list(self.tag_validators.values()):
             if validator.default and validator.name not in comment.tags:
                 comment.tags[validator.name] = \
                     Tag(name=validator.name,
@@ -322,30 +321,30 @@ def __get_formatter(self, extension_name):
 
     def __check_initial_args(self, args):
         if args.version:
-            print VERSION
+            print(VERSION)
         elif args.makefile_path:
             here = os.path.dirname(__file__)
             path = os.path.join(here, '..', 'utils', 'hotdoc.mk')
-            print os.path.abspath(path)
+            print(os.path.abspath(path))
         elif args.get_conf_path:
             key = args.get_conf_path
             path = self.config.get_path(key, rel_to_cwd=True)
             if path is not None:
-                print path
+                print(path)
         elif args.get_conf_key:
             key = args.get_conf_key
             value = self.config.get(args.get_conf_key, None)
             if value is not None:
-                print value
+                print(value)
         elif args.get_private_folder:
-            print os.path.relpath(self.__private_folder,
-                                  self.config.get_invoke_dir())
+            print(os.path.relpath(self.__private_folder,
+                                  self.config.get_invoke_dir()))
         elif args.has_extension:
             ext_name = args.has_extension
-            print ext_name in self.__extension_classes
+            print(ext_name in self.__extension_classes)
         elif args.list_extensions:
             for ext_name in self.__extension_classes:
-                print ext_name
+                print(ext_name)
         else:
             self.parser.print_usage()
 
@@ -479,7 +478,7 @@ def load_conf_file(self, conf_file, overrides):
         actual_args = {}
         defaults = {'output_format': 'html'}
 
-        for key, value in overrides.items():
+        for key, value in list(overrides.items()):
             if key in ('cmd', 'conf_file', 'dry'):
                 continue
             if value != self.parser.get_default(key):
@@ -494,7 +493,7 @@ def load_conf_file(self, conf_file, overrides):
         index = self.config.get_index()
 
         if index:
-            hash_obj = hashlib.md5(self.config.get_index())
+            hash_obj = hashlib.md5(self.config.get_index().encode('utf-8'))
             priv_name = 'hotdoc-private-' + hash_obj.hexdigest()
         else:
             priv_name = 'hotdoc-private'
@@ -524,14 +523,14 @@ def __setup_private_folder(self):
             os.mkdir(folder)
 
     def __create_extensions(self, extra_paths):
-        for ext_class in self.__extension_classes.values():
+        for ext_class in list(self.__extension_classes.values()):
             ext = ext_class(self)
             self.extensions[ext.extension_name] = ext
 
         extra_classes = get_extra_extension_classes(extra_paths)
         self.__extension_classes.update(extra_classes)
 
-        for ext_class in extra_classes.values():
+        for ext_class in list(extra_classes.values()):
             ext = ext_class(self)
             self.extensions[ext.extension_name] = ext