find-german-comments

#!/usr/bin/env python3
########################################################################
#
#  Copyright (c) 2010 Jonas Jensen, Miklos Vajna
#
#  Permission is hereby granted, free of charge, to any person
#  obtaining a copy of this software and associated documentation
#  files (the "Software"), to deal in the Software without
#  restriction, including without limitation the rights to use,
#  copy, modify, merge, publish, distribute, sublicense, and/or sell
#  copies of the Software, and to permit persons to whom the
#  Software is furnished to do so, subject to the following
#  conditions:
#
#  The above copyright notice and this permission notice shall be
#  included in all copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
#  OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
#  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
#  HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
#  WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
#  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
#  OTHER DEALINGS IN THE SOFTWARE.
#
########################################################################


import sys
import re
import subprocess
import os
import argparse
import string

class Parser:
    """
    This parser extracts comments from source files, tries to guess
    their language and then prints out the German ones.
    """
    def __init__(self):
        self.strip = string.punctuation + " \n"
        self.text_cat = self.start_text_cat()
        parser = argparse.ArgumentParser(description='Searches for German comments in cxx/hxx source files inside a given root directory recursively.')
        parser.add_argument("-f", "--filenames-only", action="store_true",
            help="Only print the filenames of files containing German comments")
        parser.add_argument("-v", "--verbose", action="store_true",
            help="Turn on verbose mode (print only positives progress to stderr)")
        parser.add_argument("-l", "--line-numbers", action="store_true",
            help="Prints the filenames and line numbers only.")
        parser.add_argument("-L", "--line-numbers-pos", action="store_true",
            help="Prints the filenames and line numbers only (if positive).")
        parser.add_argument("-t", "--threshold", action="store", default=0, type=int,
            help="When used with '--line-numbers', only bothers outputting comment info if there are more than X number of flagged comments. Useful for weeding out false positives.")
        parser.add_argument("directory", nargs='?', default='.', type=str, help='Give a directory to search in')
        self.args = parser.parse_args()
        self.check_source_files(self.args.directory)

    def get_comments(self, filename):
        """
        Extracts the source code comments.
        """
        linenum = 0
        if self.args.verbose:
            print("processing file '%s'...\n" % filename)
        sock = open(filename)
        # add an empty line to trigger the output of collected oneliner
        # comment group
        lines = sock.readlines() + ["\n"]
        sock.close()

        in_comment = False
        buf = []
        count = 1
        for i in lines:
            if "//" in i and not in_comment:
                # if we find a new //-style comment, then we
                # just append it to a previous one if: there is
                # only whitespace before the // mark that is
                # necessary to make comments longer, giving
                # more reliable output
                if not len(re.sub("(.*)//.*", r"\1", i).strip(self.strip)):
                    s = re.sub(".*// ?", "", i).strip(self.strip)
                    if len(s):
                        buf.append(s)
                else:
                    # otherwise it's an independent //-style comment in the next line
                    yield (count, "\n    ".join(buf))
                    buf = [re.sub(".*// ?", "", i.strip(self.strip))]
            elif "//" not in i and not in_comment and len(buf) > 0:
                # first normal line after a // block
                yield (count, "\n    ".join(buf))
                buf = []
            elif "/*" in i and "*/" not in i and not in_comment:
                # start of a real multiline comment
                in_comment = True
                linenum = count
                s = re.sub(r".*/\*+", "", i.strip(self.strip))
                if len(s):
                    buf.append(s.strip(self.strip))
            elif in_comment and not "*/" in i:
                # in multiline comment
                s = re.sub(r"^( |\|)*\*?", "", i)
                if len(s.strip(self.strip)):
                    buf.append(s.strip(self.strip))
            elif "*/" in i and in_comment:
                # end of multiline comment
                in_comment = False
                s = re.sub(r"\*+/.*", "", i.strip(self.strip))
                if len(s):
                    buf.append(s)
                yield (count, "\n    ".join(buf))
                buf = []
            elif "/*" in i and "*/" in i:
                # c-style oneliner comment
                yield (count, re.sub(r".*/\*(.*)\*/.*", r"\1", i).strip(self.strip))
            count += 1

    def start_text_cat(self):
        cwd = os.getcwd()
        # change to our directory
        os.chdir(os.path.split(os.path.abspath(sys.argv[0]))[0])
        sock = subprocess.Popen(["text_cat/text_cat", "-s", "-d", "text_cat/LM"], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
        os.chdir(cwd)
        return sock

    def get_lang(self, s):
        """ the output is 'german' or 'english' or 'german or english'. When
        unsure, just don't warn, there are strings where you just can't
        determine the results reliably, like '#110680#' """

        self.text_cat.stdin.write(bytes(s, 'utf-8'))
        self.text_cat.stdin.write(bytes("\n", 'utf-8'))
        self.text_cat.stdin.flush()
        lang = self.text_cat.stdout.readline().strip()
        return lang

    def is_german(self, s):
        """
        determines if a string is German or not
        """
        # for short strings we can't do reliable recognition, so skip
        # short strings and less than 4 words
        s = s.replace('\n', ' ')
        if len(s) < 32 or len(s.split()) < 4:
            return False
        return self.get_lang(s) == b"german"

    def check_file(self, path):
        """
        checks each comment in a file
        """
        def tab_calc(path):
            START = 40 #Default of 10 tabs
            if len(path) >= START:
                return 1
            diff = START - len(path)
            if diff % 4 is not 0:
                padding = 1
            else:
                padding = 0
            return (diff/4)+padding

        if self.args.line_numbers or self.args.line_numbers_pos:
            TABS = "\t"*10
            path_linenums = []
            for linenum, s in self.get_comments(path):
                if self.is_german(s):
                    path_linenums.append(linenum)
            valid = len(path_linenums) > int(self.args.threshold)
            if self.args.line_numbers:
                print("%s ... %s positives -- %s\n" % (path, str(len(path_linenums)), str(valid)))
            if valid:
                if self.args.line_numbers_pos:
                    print("%s ... %s positives\n" % (path, str(len(path_linenums))))
                    return
                if len(path) + (len(path_linenums)*4) > 75:
                    print("%s:\n" % path)
                    while path_linenums:
                        i = 0
                        numline = []
                        while i < 10:
                            try:
                                numline.append(path_linenums[0])
                                path_linenums.remove(path_linenums[0])
                            except IndexError:
                                i = 10
                            i += 1
                        numline = [str(i) for i in numline]
                        print("%s%s" % (TABS, ",".join(numline)))
                else:
                    if self.args.line_numbers:
                        path_linenums = [str(i) for i in path_linenums]
                        print("%s:%s%s" % (path, "\t"*int(tab_calc(path)), ",".join(path_linenums)))

        elif not self.args.filenames_only:
            for linenum, s in self.get_comments(path):
                if self.is_german(s):
                    print("%s:%s: %s" % (path, linenum, s))
        else:
            fnames = set([])
            for linenum, s in self.get_comments(path):
                if self.is_german(s):
                    # Make sure we print each filename only once
                    fnames.add(path)
            # Print the filenames
            for f in fnames:
                print(f)

    def first_elem(self, path):
        """
        Returns the root directory in our repo of a given path, so we can check against the allowlist.
        """
        lastElem = os.path.dirname(path)
        done = False
        while not done:
            nextElem = os.path.split(lastElem)[0]
            if nextElem is not '':
                lastElem = nextElem
            else:
                done = True
        return lastElem

    def check_source_files(self, directory):
        """
        checks each _tracked_ file in a directory recursively
        """

	# top-level project directory -> use allowlist.
        globalscan = False
        if os.path.exists(directory + "/.git/config"):
           globalscan = True

        # Change into the given dir, so "git ls-tree" does work.
        os.chdir(directory)

        sock = os.popen(r"git ls-tree -r HEAD --name-only | grep -E '\.(c|cc|cpp|cxx|h|hxx|mm)$'")
        lines = sock.readlines()
        sock.close()

        # Helps to speedup a global scan
        directory_allowlist = {
            "ure" : 1,
            "ios" : 1,
            "bean" : 1,
            "apple_remote" : 1,
            "UnoControls" : 1,
            "accessibility" : 1,
            "android" : 1,
            "animations" : 1,
            "avmedia" : 1,
            "basctl" : 1,
            "basegfx" : 1,
            "basic" : 1,
            "binaryurp" : 1,
            "bridges" : 1,
            "canvas" : 1,
            "chart2" : 1,
            "cli_ure" : 1,
            "codemaker" : 1,
            "comphelper" : 1,
            "compilerplugins" : 1,
            "configmgr" : 1,
            "connectivity" : 1,
            "cppcanvas" : 1,
            "cppu" : 1,
            "cppuhelper" : 1,
            "cpputools" : 1,
            "cui" : 1,
            "dbaccess" : 1,
            "desktop" : 1,
            "drawinglayer" : 1,
            "editeng" : 1,
            "embeddedobj" : 1,
            "embedserv" : 1,
            "eventattacher" : 1,
            "extensions" : 1,
            "external" : 1,
            "filter" : 1,
            "forms" : 1,
            "formula" : 1,
            "fpicker" : 1,
            "framework" : 1,
            "helpcompiler" : 1,
            "hwpfilter" : 1,
            "i18npool" : 1,
            "i18nlangtag" : 1,
            "i18nutil" : 1,
            "idl" : 1,
            "idlc" : 1,
            "include" : 1,
            "io" : 1,
            "javaunohelper" : 1,
            "jvmaccess" : 1,
            "jvmfwk" : 1,
            "jurt" : 1,
            "l10ntools" : 1,
            "libreofficekit" : 1,
            "lingucomponent" : 1,
            "linguistic" : 1,
            "lotuswordpro" : 1,
            "mysqlc" : 1,
            "o3tl" : 1,
            "odk" : 1,
            "officecfg" : 1,
            "opencl" : 1,
            "oox" : 1,
            "package" : 1,
            "postprocess" : 1,
            "pyuno" : 1,
            "registry" : 1,
            "remotebridges" : 1,
            "reportdesign" : 1,
            "rsc" : 1,
            "sal" : 1,
            "salhelper" : 1,
            "sax" : 1,
            "sc" : 1,
            "scaddins" : 1,
            "sccomp" : 1,
            "scripting" : 1,
            "sd" : 1,
            "sdext" : 1,
            "sfx2" : 1,
            "shell" : 1,
            "setup_native" : 1,
            "sot" : 1,
            "slideshow" : 1,
            "smoketest" : 1,
            "solenv" : 1,
            "soltools" : 1,
            "starmath" : 1,
            "stoc" : 1,
            "store" : 1,
            "svgio" : 1,
            "svl" : 1,
            "svtools" : 1,
            "svx" : 1,
            "sw" : 1,
            "test" : 1,
            "testtools" : 1,
            "toolkit" : 1,
            "tools" : 1,
            "touch" : 1,
            "ucb" : 1,
            "ucbhelper" : 1,
            "unodevtools" : 1,
            "unotest" : 1,
            "unoidl" : 1,
            "unotools" : 1,
            "unoxml" : 1,
            "uui" : 1,
            "vbahelper" : 1,
            "vcl" : 1,
            "winaccessibility" : 1,
            "writerperfect" : 1,
            "xmlhelp" : 1,
            "xmloff" : 1,
            "xmlreader" : 1,
            "xmlsecurity" : 1,
            "xmlscript" : 1,
        }

        if globalscan:
            print("Scanning all files globally:")
        elif directory == '.':
            print("Scanning all files in our current directory:")
        else:
            print("Scanning all files in", directory + ":")

        num_checked = 0

        for path in lines:
            baseDir = self.first_elem(path)
            # If we have a globalscan use the allowlist.
            if globalscan:
                if not baseDir in directory_allowlist:
                    sys.stderr.write("\n - Error: Missing path %s -\n\n" % baseDir)
                    sys.exit(1)
                elif directory_allowlist[baseDir] is 0:
                    self.check_file(path.strip())
                    num_checked = num_checked + 1
                elif directory_allowlist[baseDir] is 1:
                    sys.stderr.write("Skipping excluded directory %s\n" % baseDir)
                    directory_allowlist[baseDir] = 2
            elif not globalscan:
                self.check_file(path.strip())
                num_checked = num_checked + 1

        print("Scanned %s files\n" % num_checked)

try:
    Parser()
except KeyboardInterrupt:
    print("Interrupted!")
    sys.exit(0)

# vim:set shiftwidth=4 softtabstop=4 expandtab: