Add pydocstyle linting

And clean up any issues discovered in the files.

batchupload/helpers.py

@@ -1,8 +1,6 @@
 #!/usr/bin/python
 # -*- coding: utf-8  -*-
-"""
-Helper tools related to batchUploads
-"""
+"""Helper tools related to batchUploads."""
 import operator
 import sys  # needed by convertFromCommandline()
 import locale  # needed by convertFromCommandline()
@@ -58,10 +56,11 @@ def sortedDict(ddict):
 
 def addOrIncrement(dictionary, val, key=None):
     """
-    Add a value to the dictionary or increments the
-    counter for the value.
+    Add a value to the dictionary or increments the counter for the value.
 
-    param key: the key holding the counter
+    @param dictionary: the dictionary to update
+    @param val: the value to look for in the dictionary
+    @param key: the key holding the counter
     """
     if val not in dictionary.keys():
         if key:
@@ -124,12 +123,16 @@ def cleanString(text):
 
 def touchup(text, delimiter=None, delimiter_replacement=None):
     """
-    Tweaks a string by removing surrounding bracket or quotes as well as
-    some trailing punctuation.
+    Perform various cleanup processes on a string.
+
+    Tweaks a string by:
+    * removing surrounding bracket or quotes
+    * remove some trailing punctuation.
 
     @param text: the text to touch up
     @param delimiter: a delimiter to replace
     @param delimiter_replacement: what to replace the delimiter by
+    @return string
     """
     delimiter_replacement = delimiter_replacement or ', '
 
@@ -156,10 +159,12 @@ def touchup(text, delimiter=None, delimiter_replacement=None):
 
 
 def shortenString(text):
-    '''
-    If a string is larger than GOODLENGTH then this tries to
-    find a sensibel shortening.
-    '''
+    """
+    Shorten strings longer than GOODLENGTH.
+
+    @param text: the text to shorten
+    @return string
+    """
     badchar = u'-., '  # maybe also "?
     if u'<!>' in text:
         text = text[:text.find(u'<!>')]

batchupload/listscraper.py

@@ -1,5 +1,6 @@
 #!/usr/bin/python
 # -*- coding: utf-8  -*-
+"""Toolkit for scraping (wikitext) lists on a wiki."""
 #
 # Tool for scraping existing wiki lists from commons and
 # storing these as correctly formatted local files
@@ -25,10 +26,10 @@ def parseEntries(contents,
     For non-empty entries a list of values is always returned.
     "<small>"-tags are striped from the input.
 
-    :param content: wikicode
-    :param row_t: row template
-    :default_params: dict of expected params and their default values
-    :return list: of entry-dict items
+    @param content: wikicode
+    @param row_t: row template
+    @param default_params: dict of expected params and their default values
+    @return list: of entry-dict items
     """
     default_params = default_params or {
         u'name': '',
@@ -58,15 +59,17 @@ def parseEntries(contents,
     return units
 
 
-def formatEntry(u, typ=u'category'):
+def formatEntry(unit, typ=u'category'):
     """
-    Given an mapping unit remove skipped entries, leave only category as a list
-    and make frequency a number.
+    Extract a given value from a list together with its frequency.
 
-    :param typ: which parameter to return (defaults to "category")
+    Given an mapping unit remove skipped entries, leave only the named entry
+    as a list and make frequency a number.
+
+    @param typ: which parameter to return (defaults to "category")
     """
     # remove any -, make frequency and int
-    for k, v in u.iteritems():
+    for k, v in unit.iteritems():
         # handle lists
         if k == typ:
             if v == '':
@@ -82,28 +85,28 @@ def formatEntry(u, typ=u'category'):
                 v = ''
             elif isinstance(v, list) and v[0] == '-':
                 v = []
-        u[k] = v
-    return u
+        unit[k] = v
+    return unit
 
 
 def scrape(pages, prefix, working_path=None, out_path=None, site=None):
     """
     Scrape lists on commons and overwrite local files.
 
-    :param pages: A mapping of Commons pages to output files
+    @param pages: A mapping of Commons pages to output files
         where Commons pages get the format prefix*
         and output file the format: commons-*.json
         example: {u'People': u'People',
                   u'Keywords': u'Keywords',
                   u'Materials': u'Materials',
                   u'Places': u'Places'}
 
-    :param prefix: prefix under which lists are found
+    @param prefix: prefix under which lists are found
         example: u'Commons:Batch uploading/LSH'
-    :param working_path: path to directory in which to work (if not current)
+    @param working_path: path to directory in which to work (if not current)
         modifies out_path
-    :param out_path: path to directory in which output files are put
-    :param site: pywikibot.site object, default Commons
+    @param out_path: path to directory in which output files are put
+    @param site: pywikibot.site object, default Commons
     """
     out_path = out_path or OUT_PATH
 
@@ -135,13 +138,13 @@ def mergeWithOld(sorted_dict, pagename, output_wiki,
     """
     Output mapping lists in wiki format, merging with any existing.
 
-    :param sorted_dict prefix under which lists are found
+    @param sorted_dict prefix under which lists are found
         example: u'Commons:Batch uploading/LSH'
-    :param pagename: name of the list
-    :param output_wiki: method for outputting wikitext
-    :param working_path: path to directory in which to work (if not current)
+    @param pagename: name of the list
+    @param output_wiki: method for outputting wikitext
+    @param working_path: path to directory in which to work (if not current)
         modifies out_path
-    :param out_path: path to directory in which output files are put
+    @param out_path: path to directory in which output files are put
     """
     out_path = out_path or OUT_PATH
 
@@ -188,9 +191,9 @@ def makeEntry(name, frequency, previous=None):
     Create a list entry in the relevant format.
 
     It is either created from scratch or by reusing mappings.
-    :param frequency: frequency of the entry
-    :param previous: previous mapping for the entry
-    :return: entry
+    @param frequency: frequency of the entry
+    @param previous: previous mapping for the entry
+    @return: entry
     """
     if frequency > 0:
         if previous:

batchupload/make_info.py

@@ -1,9 +1,9 @@
 #!/usr/bin/python
 # -*- coding: utf-8  -*-
 """
-Abstract class for producing mapping tables and file description pages
+Abstract class for producing mapping tables and file description pages.
 
-TODO: add an entry point to make/update mappings
+@TODO: add an entry point to make/update mappings
 """
 import batchupload.common as common
 import batchupload.helpers as helpers
@@ -38,7 +38,6 @@ def make_info_page(data):
 
 
 class MakeBaseInfo(object):
-
     """Abstract class for generating descriptions and filenames for a batch."""
 
     __metaclass__ = ABCMeta
@@ -59,11 +58,13 @@ def __init__(self, base_meta_cat, batch_label):
     @abstractmethod
     def load_data(self, in_file):
         """
-        Load the provided data (in whichever format) and produce a dict with an
-        entry per file which can be used for further processing.
+        Load the provided data and make suitable for input to process_data().
+
+        The provided data can be in any format and include more than one file.
+        The output format can likewise be anything which is accepted by
+        process_data().
 
         @param in_file: the path to the metadata file or list of such paths
-        @return: dict
         """
         pass
 
@@ -83,13 +84,11 @@ def load_mappings(self, update=True):
     @abstractmethod
     def process_data(self, raw_data):
         """
-        Process the raw data from load_data into the format usable by
-        make_info_template.
+        Process the output of load_data into a format usable by make_info().
 
         The processed data is stored in self.data, a dict of items or objects.
 
         @param raw_data: output from load_data()
-        @return: dict
         """
         pass
 

batchupload/prepUpload.py

@@ -19,10 +19,10 @@ def run(in_path, out_path, data_path, file_exts=None):
         1. Finds files in inpath (with subdirs) with the right file extension,
         2. Matching these against the keys in the makeInfo output data
         3. Making info files and renaming found file (in new target folder)
-    :param in_path: path to directory where unprocessed files live
-    :param outPath: path to directory where renamed files and info should live
-    :param dataPath: path to .json containing makeInfo output data
-    :param file_exts: tupple of allowed file extensions (case insensitive)
+    @param in_path: path to directory where unprocessed files live
+    @param outPath: path to directory where renamed files and info should live
+    @param dataPath: path to .json containing makeInfo output data
+    @param file_exts: tupple of allowed file extensions (case insensitive)
 
     @todo: throw errors on failed file read/write
     """
@@ -52,10 +52,10 @@ def find_files(path, file_exts, subdir=True):
     """
     Identify all files with a given extension in a given directory.
 
-    :param path: path to directory to look in
-    :param file_exts: tuple of allowed file extensions (case insensitive)
-    :param subdir: whether subdirs should also be searched
-    :return: list of paths to found files
+    @param path: path to directory to look in
+    @param file_exts: tuple of allowed file extensions (case insensitive)
+    @param subdir: whether subdirs should also be searched
+    @return: list of paths to found files
     """
     files = []
     subdirs = []
@@ -72,10 +72,17 @@ def find_files(path, file_exts, subdir=True):
 
 def makeHitlist(files, data):
     """
-    Given a list of paths to files extract the extension (lower case) and the
-    extensionless basename.
-    param files: list of file paths
-    return list of hitList[key] = {ext, path, data}
+    Given a list of paths to file and target filenames construct a hitlist.
+
+    The hitlist is made up by the (lower case) extension and the
+    extensionless basename of the file.
+
+    The data file should be a dict where the keys are the (extensionless)
+    target filenames.
+
+    @param files: list of file paths
+    @param data: dict containing target filenames as keys
+    @return: list of hitList[key] = {ext, path, data}
     """
     hitlist = []
     processed_keys = []  # stay paranoid
@@ -95,8 +102,8 @@ def makeAndRename(hitlist, outPath):
     """
     Given a hitlist create the info files and and rename the matched file.
 
-    param hitlist: the output of makeHitlist
-    param outPath: the directory in which to store info + renamed files
+    @param hitlist: the output of makeHitlist
+    @param outPath: the directory in which to store info + renamed files
     """
     # create outPath if it doesn't exist
     common.create_dir(outPath)
@@ -125,8 +132,8 @@ def removeEmptyDirectories(path, top=True):
     """
     Remove any empty directories and subdirectories.
 
-    :param path: path to directory to start deleting from
-    :param top: set to True to not delete the starting directory
+    @param path: path to directory to start deleting from
+    @param top: set to True to not delete the starting directory
     """
     if not os.path.isdir(path):
         return

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = flake8,py27
+envlist = flake8,pydocstyle,py27
 skipsdist=true
 
 [testenv]
@@ -14,5 +14,13 @@ deps = flake8
 commands = flake8
 
 [flake8]
-exclude = user-config.py,.venv,.tox,./KB-maps/,./KB-maps-upload/,./Svenska_vapen/
+exclude = user-config.py,.venv,.tox,./Batches/
 ignore = E501,F841
+
+[testenv:pydocstyle]
+deps = pydocstyle
+commands = pydocstyle
+
+[pydocstyle]
+match-dir = batchupload
+match = (?![test_|__init__|user\-config]).*\.py