Permalink
Browse files

Port to python 3

  • Loading branch information...
MathieuDuponchelle committed Nov 22, 2016
1 parent cd9bfdb commit 385d20d0f9ca12ad1591f6e0478fce721406fcee
Showing with 612 additions and 530 deletions.
  1. +7 −10 hotdoc/core/base_extension.py
  2. +4 −5 hotdoc/core/base_formatter.py
  3. +15 −16 hotdoc/core/change_tracker.py
  4. +1 −2 hotdoc/core/comment_block.py
  5. +4 −5 hotdoc/core/config.py
  6. +1 −2 hotdoc/core/doc_database.py
  7. +24 −25 hotdoc/core/doc_repo.py
  8. +27 −27 hotdoc/core/doc_tree.py
  9. +3 −3 hotdoc/core/exceptions.py
  10. +0 −1 hotdoc/core/file_includer.py
  11. +7 −9 hotdoc/core/links.py
  12. +1 −1 hotdoc/core/symbols.py
  13. +0 −1 hotdoc/core/tests/test_config_parser.py
  14. +1 −2 hotdoc/core/tests/test_doc_tree.py
  15. +0 −1 hotdoc/core/tests/test_links.py
  16. +1 −2 hotdoc/core/tests/test_page.py
  17. +46 −34 hotdoc/extensions/devhelp/devhelp_extension.py
  18. +74 −65 hotdoc/extensions/license/license_extension.py
  19. +79 −60 hotdoc/extensions/search/create_index.py
  20. +18 −13 hotdoc/extensions/search/search_extension.py
  21. +45 −34 hotdoc/extensions/search/trie.py
  22. +0 −1 hotdoc/extensions/syntax_highlighting/syntax_extension.py
  23. +40 −29 hotdoc/extensions/tags/tag_extension.py
  24. +8 −8 hotdoc/formatters/html_formatter.py
  25. +1 −1 hotdoc/formatters/html_templates/callable.html
  26. +1 −1 hotdoc/formatters/html_templates/parameter_detail.html
  27. +1 −1 hotdoc/formatters/html_templates/property.html
  28. +1 −1 hotdoc/formatters/html_templates/return_item.html
  29. +0 −9 hotdoc/hotdoc
  30. +78 −44 hotdoc/parsers/cmark_module.c
  31. +1 −3 hotdoc/parsers/cmark_utils.py
  32. +5 −7 hotdoc/parsers/gtk_doc_parser.py
  33. +5 −3 hotdoc/parsers/standalone_parser.py
  34. +4 −4 hotdoc/run_hotdoc.py
  35. +3 −3 hotdoc/utils/alchemy_integration.py
  36. +9 −6 hotdoc/utils/loggable.py
  37. +3 −3 hotdoc/utils/patcher.py
  38. +2 −2 hotdoc/utils/setup_utils.py
  39. +2 −3 hotdoc/utils/simple_signals.py
  40. +3 −2 hotdoc/utils/tests/test_loggable.py
  41. +23 −24 hotdoc/utils/utils.py
  42. +1 −1 pre-commit
  43. +63 −56 setup.py
@@ -19,7 +19,6 @@
"""
Utilities and baseclasses for extensions
"""
import os
from collections import defaultdict
@@ -101,7 +100,7 @@ def __init__(self, doc_repo):
if not hasattr(self, 'formatters'):
self.formatters = {"html": HtmlFormatter([])}
self.__created_symbols = defaultdict(OrderedSet)
self._created_symbols = defaultdict(OrderedSet)
self.__package_root = None
# pylint: disable=no-self-use
@@ -168,7 +167,7 @@ def reset(self):
This function is only useful for testing purposes, at least
for now.
"""
self.__created_symbols = defaultdict(OrderedSet)
self._created_symbols = defaultdict(OrderedSet)
self.__package_root = None
def setup(self):
@@ -224,11 +223,11 @@ def parse_standard_config(cls, config):
cls.smart_index = bool(config.get('%s_smart_index' %
cls.argument_prefix))
for arg, dest in cls.paths_arguments.items():
for arg, dest in list(cls.paths_arguments.items()):

This comment has been minimized.

Show comment
Hide comment
@andrewshadura

andrewshadura Jan 27, 2017

Is list() really necessary here? I thought for worked fine with iterators…

@andrewshadura

andrewshadura Jan 27, 2017

Is list() really necessary here? I thought for worked fine with iterators…

This comment has been minimized.

Show comment
Hide comment
@pwithnall

pwithnall Jan 27, 2017

Contributor

I assume this is 2to3 being over-cautious.

@pwithnall

pwithnall Jan 27, 2017

Contributor

I assume this is 2to3 being over-cautious.

This comment has been minimized.

Show comment
Hide comment
@andrewshadura

andrewshadura Jan 27, 2017

Aha, I never used 2to3, I only made some attempts to manually port some of my code to Python 3. I should port the rest too.

@andrewshadura

andrewshadura Jan 27, 2017

Aha, I never used 2to3, I only made some attempts to manually port some of my code to Python 3. I should port the rest too.

This comment has been minimized.

Show comment
Hide comment
@MathieuDuponchelle

MathieuDuponchelle Jan 27, 2017

Collaborator

I indeed used 2to3 for this, @andrewshadura any reason for you to react on that specific line ?

@MathieuDuponchelle

MathieuDuponchelle Jan 27, 2017

Collaborator

I indeed used 2to3 for this, @andrewshadura any reason for you to react on that specific line ?

This comment has been minimized.

Show comment
Hide comment
@andrewshadura

andrewshadura Jan 28, 2017

No, I was wondering how much porting was needed, and I noticed this change in many places, so I just commented on one of them :)

@andrewshadura

andrewshadura Jan 28, 2017

No, I was wondering how much porting was needed, and I noticed this change in many places, so I just commented on one of them :)

This comment has been minimized.

Show comment
Hide comment
@MathieuDuponchelle

MathieuDuponchelle Jan 30, 2017

Collaborator

ack

@MathieuDuponchelle
val = config.get_paths(arg)
setattr(cls, dest, val)
for arg, dest in cls.path_arguments.items():
for arg, dest in list(cls.path_arguments.items()):
val = config.get_path(arg)
setattr(cls, dest, val)
@@ -349,11 +348,9 @@ def get_or_create_symbol(self, *args, **kwargs):
symbols.Symbol: the created symbol, or `None`.
"""
sym = self.doc_repo.doc_database.get_or_create_symbol(*args, **kwargs)
# pylint: disable=unidiomatic-typecheck
if sym and type(sym) != Symbol and sym.filename:
# assert sym.filename is not None
self.__created_symbols[sym.filename].add(sym.unique_name)
self._created_symbols[sym.filename].add(sym.unique_name)
return sym
@@ -387,12 +384,12 @@ def __update_doc_tree_cb(self, doc_tree, unlisted_sym_names):
for sym_name in unlisted_sym_names:
sym = self.doc_repo.doc_database.get_symbol(sym_name)
if sym and sym.filename in self._get_all_sources():
self.__created_symbols[sym.filename].add(sym_name)
self._created_symbols[sym.filename].add(sym_name)
user_pages = [p for p in doc_tree.walk(index) if not p.generated]
user_symbols = self.__get_user_symbols(user_pages)
for source_file, symbols in self.__created_symbols.items():
for source_file, symbols in list(self._created_symbols.items()):
gen_symbols = symbols - user_symbols
self.__add_subpage(doc_tree, index, source_file, gen_symbols)
doc_tree.stale_symbol_pages(symbols)
@@ -19,24 +19,24 @@
"""
This module defines a base Formatter class
"""
import os
import shutil
from collections import defaultdict
from schema import Schema, Optional
import pygraphviz as pg
from hotdoc.core.doc_tree import Page
from hotdoc.utils.utils import symlink
import pygraphviz as pg
from hotdoc.utils.configurable import Configurable
from hotdoc.utils.simple_signals import Signal
from hotdoc.utils.utils import recursive_overwrite, OrderedSet
Page.meta_schema[Optional('extra', default=defaultdict())] = \
Schema({unicode: object})
Schema({str: object})
def _create_hierarchy_graph(hierarchy):
@@ -166,8 +166,7 @@ def write_page(self, page, root, output):
self.writing_page_signal(self, page, path)
with open(path, 'w') as _:
out = page.detailed_description
_.write(out.encode('utf-8'))
_.write(page.detailed_description)
self.__copy_extra_files(root, os.path.dirname(path))
@@ -19,7 +19,6 @@
"""
Defines and tests ChangeTracker
"""
import os
from collections import defaultdict
@@ -82,7 +81,7 @@ def hard_dependencies_are_stale(self):
"""
Banana banana
"""
for filename, last_mtime in self.hard_deps_mtimes.items():
for filename, last_mtime in list(self.hard_deps_mtimes.items()):
mtime = get_mtime(filename)
if mtime == -1 or mtime != last_mtime:
@@ -95,31 +94,31 @@ def hard_dependencies_are_stale(self):
# Initial build
os.system('touch a b c d')
print "Should be ([a, b, c, d], [])"
print CTRACKER.get_stale_files(['a', 'b', 'c', 'd'], 'testing')
print("Should be ([a, b, c, d], [])")
print(CTRACKER.get_stale_files(['a', 'b', 'c', 'd'], 'testing'))
# Build where nothing changed
print "Should be ([], [])"
print CTRACKER.get_stale_files(['a', 'b', 'c', 'd'], 'testing')
print("Should be ([], [])")
print(CTRACKER.get_stale_files(['a', 'b', 'c', 'd'], 'testing'))
# Build with two files changed
os.system('touch b d')
print "Should be ([b, d], [])"
print CTRACKER.get_stale_files(['a', 'b', 'c', 'd'], 'testing')
print("Should be ([b, d], [])")
print(CTRACKER.get_stale_files(['a', 'b', 'c', 'd'], 'testing'))
# Build where one file was removed
os.system('rm -f b')
print "Should be ([b], [])"
print CTRACKER.get_stale_files(['a', 'b', 'c', 'd'], 'testing')
print "Should be ([], [])"
print CTRACKER.get_stale_files(['a', 'b', 'c', 'd'], 'testing')
print("Should be ([b], [])")
print(CTRACKER.get_stale_files(['a', 'b', 'c', 'd'], 'testing'))
print("Should be ([], [])")
print(CTRACKER.get_stale_files(['a', 'b', 'c', 'd'], 'testing'))
# Build where one file was unlisted
print "Should be ([], [a])"
print CTRACKER.get_stale_files(['b', 'c', 'd'], 'testing')
print("Should be ([], [a])")
print(CTRACKER.get_stale_files(['b', 'c', 'd'], 'testing'))
# Build with file listed again
print "Should be ([a], [])"
print CTRACKER.get_stale_files(['a', 'b', 'c', 'd'], 'testing')
print("Should be ([a], [])")
print(CTRACKER.get_stale_files(['a', 'b', 'c', 'd'], 'testing'))
os.system('rm -f a b c d')
@@ -20,7 +20,6 @@
This module implements a Comment class, to be used
by code-parsing extensions.
"""
import linecache
import os
from collections import defaultdict
@@ -69,7 +68,7 @@ def __init__(self, name=u'', title=None, params=None, filename=u'',
self.col_offset = 0
self.initial_col_offset = 0
self.annotations = annotations or {}
self.description = unicode(description)
self.description = str(description)
self.short_description = short_description
self.extension_attrs = defaultdict(lambda: defaultdict(dict))
self.tags = tags or {}
View
@@ -19,7 +19,6 @@
"""
Implement a high-level config parser for hotdoc.
"""
import os
import json
import glob
@@ -308,14 +307,14 @@ def get_dependencies(self):
tracked configuration.
"""
all_deps = OrderedSet()
for key, _ in self.__config.items():
for key, _ in list(self.__config.items()):
if key in self.__cli:
continue
if key.endswith('sources'):
all_deps |= self.get_sources(key[:len('sources') * -1])
for key, _ in self.__cli.items():
for key, _ in list(self.__cli.items()):
if key.endswith('sources'):
all_deps |= self.get_sources(key[:len('sources') * -1])
@@ -344,12 +343,12 @@ def dump(self, conf_file=None):
conf_dir = self.__conf_dir
final_conf = {}
for key, value in self.__config.items():
for key, value in list(self.__config.items()):
if key in self.__cli:
continue
final_conf[key] = value
for key, value in self.__cli.items():
for key, value in list(self.__cli.items()):
if (key.endswith('index') and not key.endswith('smart_index')) or \
key in ['sitemap', 'output']:
path = self.__abspath(value, from_conf=False)
@@ -18,7 +18,6 @@
"""Banana banana
"""
import os
# pylint: disable=import-error
@@ -91,7 +90,7 @@ def get_or_create_symbol(self, type_, **kwargs):
debug('Created symbol with unique name %s' % unique_name,
'symbols')
for key, value in kwargs.items():
for key, value in list(kwargs.items()):
setattr(symbol, key, value)
if self.__incremental:
View
@@ -19,15 +19,16 @@
"""
Core of the core.
"""
import argparse
import hashlib
import cPickle as pickle
import pickle as pickle
import os
import shutil
import sys
import io
from collections import OrderedDict
from hotdoc.core import file_includer
from hotdoc.core.base_extension import BaseExtension
from hotdoc.core.change_tracker import ChangeTracker
@@ -117,8 +118,8 @@ def __init__(self):
self.datadir = "/usr/share"
self.__conf_file = None
self.__extension_classes = {
CoreExtension.extension_name: CoreExtension}
self.__extension_classes = OrderedDict({
CoreExtension.extension_name: CoreExtension})
self.__index_file = None
self.__root_page = None
self.__base_doc_folder = None
@@ -149,8 +150,8 @@ def patch_page(self, symbol, raw_comment):
"""
Banana banana
"""
pages = self.doc_tree.get_pages_for_symbol(symbol.unique_name)
if not pages:
page = self.doc_tree.get_page_for_symbol(symbol.unique_name)
if not page:
return False
old_comment = symbol.comment
@@ -168,11 +169,9 @@ def patch_page(self, symbol, raw_comment):
if new_comment.name != symbol.unique_name:
return False
pages = pages.values()
symbol.comment = new_comment
for page in pages:
formatter = self.__get_formatter(page.extension_name)
formatter.patch_page(page, symbol, self.output)
formatter = self.__get_formatter(page.extension_name)
formatter.patch_page(page, symbol, self.output)
return True
@@ -224,7 +223,7 @@ def setup(self):
self.doc_tree = DocTree(self.get_private_folder(), self.include_paths)
for extension in self.extensions.values():
for extension in list(self.extensions.values()):
info('Setting up %s' % extension.extension_name)
extension.setup()
self.doc_database.flush()
@@ -271,7 +270,7 @@ def __dump_deps_file(self):
_.write(u'%s ' % dep)
if self.doc_tree:
for page in self.doc_tree.get_pages().values():
for page in list(self.doc_tree.get_pages().values()):
if not page.generated:
empty_targets.append(page.source_file)
_.write(u'%s ' % page.source_file)
@@ -280,7 +279,7 @@ def __dump_deps_file(self):
_.write(u'\n\n%s:' % empty_target)
def __add_default_tags(self, _, comment):
for validator in self.tag_validators.values():
for validator in list(self.tag_validators.values()):
if validator.default and validator.name not in comment.tags:
comment.tags[validator.name] = \
Tag(name=validator.name,
@@ -322,30 +321,30 @@ def __get_formatter(self, extension_name):
def __check_initial_args(self, args):
if args.version:
print VERSION
print(VERSION)
elif args.makefile_path:
here = os.path.dirname(__file__)
path = os.path.join(here, '..', 'utils', 'hotdoc.mk')
print os.path.abspath(path)
print(os.path.abspath(path))
elif args.get_conf_path:
key = args.get_conf_path
path = self.config.get_path(key, rel_to_cwd=True)
if path is not None:
print path
print(path)
elif args.get_conf_key:
key = args.get_conf_key
value = self.config.get(args.get_conf_key, None)
if value is not None:
print value
print(value)
elif args.get_private_folder:
print os.path.relpath(self.__private_folder,
self.config.get_invoke_dir())
print(os.path.relpath(self.__private_folder,
self.config.get_invoke_dir()))
elif args.has_extension:
ext_name = args.has_extension
print ext_name in self.__extension_classes
print(ext_name in self.__extension_classes)
elif args.list_extensions:
for ext_name in self.__extension_classes:
print ext_name
print(ext_name)
else:
self.parser.print_usage()
@@ -479,7 +478,7 @@ def load_conf_file(self, conf_file, overrides):
actual_args = {}
defaults = {'output_format': 'html'}
for key, value in overrides.items():
for key, value in list(overrides.items()):
if key in ('cmd', 'conf_file', 'dry'):
continue
if value != self.parser.get_default(key):
@@ -494,7 +493,7 @@ def load_conf_file(self, conf_file, overrides):
index = self.config.get_index()
if index:
hash_obj = hashlib.md5(self.config.get_index())
hash_obj = hashlib.md5(self.config.get_index().encode('utf-8'))
priv_name = 'hotdoc-private-' + hash_obj.hexdigest()
else:
priv_name = 'hotdoc-private'
@@ -524,14 +523,14 @@ def __setup_private_folder(self):
os.mkdir(folder)
def __create_extensions(self, extra_paths):
for ext_class in self.__extension_classes.values():
for ext_class in list(self.__extension_classes.values()):
ext = ext_class(self)
self.extensions[ext.extension_name] = ext
extra_classes = get_extra_extension_classes(extra_paths)
self.__extension_classes.update(extra_classes)
for ext_class in extra_classes.values():
for ext_class in list(extra_classes.values()):
ext = ext_class(self)
self.extensions[ext.extension_name] = ext
Oops, something went wrong.

0 comments on commit 385d20d

Please sign in to comment.