find-copyright

#!/usr/bin/python3

import sys
import fileinput
import magic
import re
from comment_parser import comment_parser

end_pats = (r'This implementation just',
            r'Synopsis of public',
            r'The SPU must have',
            r'This is a simple version',
            r'This is a dummy',
            r'stdio_ext\.h',
            r'python script to',
            r'libgen\.h',
            r'Id:.*Exp',
            r'sccs\.',
            r'Tests gleaned',
            r'tar\.h',
            r'tzcalc_limits\.c',
            r'dummy file',
            r'Rearranged for general',
            r'sincos',
            r'POSIX',
            r'Reentrant',
            r'Copied',
            r'These are',
            r'creat',
            r'ARM configuration',
            r'Place holder',
            r'local header',
            r'GNU variant',
            r'default reentrant',
            r'A replacement',
            r'The signgam',
            r'dummy header',
            r'Uniset',
            r'wcsftime\.c',
            r'month_lengths\.c',
            r'Static instance',
            r'Conversion is performed',
            r'Common routine',
            r'l64a')

end_pats_s = (r'FUNCTION',)

end_res = []
for pat in end_pats:
        regex = re.compile(pat, re.I)
        end_res += [regex]

for pat in end_pats_s:
        regex = re.compile(pat)
        end_res += [regex]

left_res = re.compile(r'^[^A-Za-z0-9]*(.*)')
right_res = re.compile(r'(.*)[ /*\t]$')

def clean_copyright(string):
        copyright = []
        have_cpr = False
        cpr_line = re.compile(r'copyright.*(20[0-2][0-9]|19[7-9][0-9])', re.I)
        upper = re.compile(r'[A-Z]')
        lower = re.compile(r'[a-z]')
        modified = re.compile(r'Modified')
        derived = re.compile(r'code is derived from software', re.I)
        skipping = False
        only_upper = False
        for line in string.splitlines():
                m = cpr_line.search(line)
                if m:
                        line = line[m.start():]
                        m = re.match(r'(.*<[^>]+>).*', line)
                        if m:
                                line = line[:m.end()]
                        have_cpr = True

                if not have_cpr:
                        continue

                end = False
                for regex in end_res:
                        if regex.search(line):
                                end = True
                                break
                if end:
                        break

                if modified.search(line):
                        skipping = True
                        continue
                if derived.search(line):
                        skipping = True
                        continue
                if only_upper:
                        if lower.search(line):
                                break
                elif upper.search(line) and not lower.search(line):
                        only_upper = True
                
                line = left_res.match(line).group(1)
                while True:
                        m = right_res.match(line)
                        if not m:
                                break
                        line = m.group(1)
                if skipping:
                        if len(line) == 0:
                                skipping = False
                        continue
                copyright += [line]
        t = '\n'.join(copyright).strip()
        return t

def get_license_type(copyright):
        if re.match(r'^no support for.*', copyright):
                return "Default"
        if copyright == 'none' or copyright == '':
                return "Default"
        if re.search(r'NetBSD', copyright):
                return "NetBSD"
        if (re.search(r'Redistributions *(of)? *source +code', copyright) and
            re.search(r'Redistributions +in +binary +form', copyright)):
           if re.search(r'[tT]he +names? of', copyright):
                   return "BSD3"
           else:
                   return "BSD2"
        if (re.search(r'University', copyright) and
            re.search(r'California', copyright) and
            re.search(r'Berkeley', copyright)):
                return "UCB"
        if re.search(r'FreeBSD', copyright):
                return "FreeBSD"
        if (re.search(r'AGPL', copyright) or
            re.search(r'Affero General Public License', copyright)):
                if re.search(r'version 3', copyright):
                        return "AGPL3"
                return "AGPL"
        if re.search(r'the GPL', copyright):
                if re.search(r'version 2', copyright):
                        return "GPL2"
                if re.search(r'version 3', copyright):
                        return "GPL3"
                return "GPL"
        return "Other"

def find_copyright_source(name, mime):
        with open(name, 'rb') as f:
                text = f.read().decode('utf-8', errors='ignore')
                try:
                        comments = comment_parser.extract_comments_from_str(text, mime)
                except:
                        return None
        tog = re.compile(r'.*Open Group Base Specification', re.I | re.S)
        m = re.compile(r'.*copyright.*', re.I | re.S)
        for i in range(len(comments)):
                comment = comments[i]
                if tog.match(comment.text()):
                        continue
                if m.match(comment.text()):
                        bits = comment.text()
                        line = comments[i].line_number() + 1
                        i += 1
                        while i < len(comments):
                                if comments[i].line_number() != line:
                                        break
                                bits += '\n' + comments[i].text()
                                i += 1
                                line += 1
                        return clean_copyright(bits)
        return None

# Copyright holder is generally the first paragraph of the text

def pick_split(split,match,use_end):
        if not match:
                return split
        start = match.start()
        end = match.start()
        if use_end:
                end = match.end()
        if not split:
                return (start, end)
        if start < split[0]:
                return (start, end)
        return split

def get_split(copyright):
        lastcopy = None
        for lastcopy in re.finditer(r'copyright (\(c\))? *(19[7-9][0-9]|20[0-2][0-9])[^A-Z]*[A-Z][^\n]*', copyright, re.I|re.S):
                pass
        if lastcopy:
                para = None
                for para in re.finditer(r'\n\n', copyright, re.S):
                        if para.start() >= lastcopy.end():
                                break
                all = None
                for all in re.finditer(r'all rights reserved\.?', copyright, re.I | re.S):
                        if all.start() >= lastcopy.end():
                                break
        else:
                para = re.search(r'\n\n', copyright, re.S)
                all = re.search(r'all rights reserved\.?', copyright, re.I | re.S)
        licensed = re.search(r'This ', copyright, re.I | re.S)
        portions = re.search(r'Portions', copyright, re.I | re.S)
        mod = re.search(r'modification', copyright, re.I | re.S)
        perm = re.search(r'permission to', copyright, re.I | re.S)
        public = re.search(r'public domain', copyright, re.I | re.S)

        split = None
        split = pick_split(split, all, True)
        if not split:
                split = pick_split(split, para, True)
        split = pick_split(split, licensed, False)
        split = pick_split(split, portions, False)
        split = pick_split(split, mod, False)
        split = pick_split(split, perm, False)
        split = pick_split(split, public, False)
        return split

def get_holder(copyright):
        split = get_split(copyright)
        if split:
                return copyright[:split[0]]
        return copyright

def get_license(copyright):
        split = get_split(copyright)
        if split:
                copyright = copyright[split[1]:]
                m = re.match(r'^[ \t]*\n[ \t]*\n*', copyright, re.I|re.S)
                if m:
                        copyright = copyright[m.end():]
                return copyright
        if copyright == "unknown file type":
                return copyright
        return ""

def pound_comments(name):
        l = re.compile(r'^#[ \t]*(.*)$')
        comments = []
        comment = []
        for line in open(name).readlines():
                m = l.match(line)
                if m:
                        after = m.group(1)
                        comment += [after]
                elif comment:
                        comments += ['\n'.join(comment)]
                        comment = []
        if comment:
                comments += ['\n'.join(comment)]
        return comments

def find_copyright_pound(name):
        comments = pound_comments(name)
        m = re.compile(r'.*copyright.*', re.I | re.S)
        for comment in comments:
                if m.match(comment):
                        return clean_copyright(comment)
        return None

def semi_comments(name):
        l = re.compile(r'^;;*[ \t]*(.*)$')
        comments = []
        comment = []
        for line in open(name).readlines():
                m = l.match(line)
                if m:
                        after = m.group(1)
                        comment += [after]
                elif comment:
                        comments += ['\n'.join(comment)]
                        comment = []
        if comment:
                comments += ['\n'.join(comment)]
        return comments

def find_copyright_semi(name):
        comments = semi_comments(name)
        m = re.compile(r'.*copyright.*', re.I | re.S)
        for comment in comments:
                if m.match(comment):
                        return clean_copyright(comment)
        return None

def troff_comments(name):
        l = re.compile(r'^\.\\"[ \t]*(.*)$')
        comments = []
        comment = []
        for line in open(name).readlines():
                m = l.match(line)
                if m:
                        after = m.group(1)
                        comment += [after]
                elif comment:
                        comments += ['\n'.join(comment)]
                        comment = []
        if comment:
                comments += ['\n'.join(comment)]
        return comments

def find_copyright_troff(name):
        comments = troff_comments(name)
        m = re.compile(r'.*copyright.*', re.I | re.S)
        for comment in comments:
                if m.match(comment):
                        return clean_copyright(comment)
        return None

def clean(str,chars):
        out = ""
        for c in str:
                if c not in chars:
                        out += c.lower()
        return out

def pack_copyright(copyright):
        return clean(copyright, " .,!*\n\t")

def num_lines(name):
        with open(name, 'rb') as f:
                text = f.read().decode('utf-8', errors='ignore').splitlines()
                return len(text)

def starts_with(pattern, name):
        with open(name, 'rb') as f:
                text = f.read().decode('utf-8', errors='ignore')
                m = re.search(pattern, text)
                return m and m.start() == 0

def file_contains(pattern, name):
        with open(name, 'rb') as f:
                text = f.read().decode('utf-8', errors='ignore')
                return re.search(pattern, text)

def main():
        names = []
        for name in sys.argv[1:]:
                if name == '-':
                        for line in sys.stdin:
                                names += [line.strip()]
                else:
                        names += [name]
        copyright_users = {}
        copyrights = {}
        copyright_files = {}
        for name in names:
                copyright = None

                # Data files don't need a license

                if re.match(r'.*\.t$', name):
                        continue
                if re.match(r'.*\.cct$', name):
                        continue
                if re.match(r'.*ChangeLog.*', name):
                        continue
                if re.match(r'.*COPYING.*', name):
                        continue
                if re.match(r'.*NEWS.*', name):
                        continue
                if re.match(r'.*MAINTAINERS', name):
                        continue
                if re.match(r'CODE_OF_CONDUCT.*', name):
                        continue
                if re.match(r'CONTRIBUTING.*', name):
                        continue

                if re.match(r'.*\.[ch]$', name) or re.match(r'.*\.ld$', name) or re.match(r'.*\.[ch]\.in$', name):
                        copyright = find_copyright_source(name, 'text/x-c')
                elif re.match(r'.*\.[sS]$', name):
                        copyright = find_copyright_semi(name)
                        if not copyright:
                                copyright = find_copyright_source(name, 'text/x-c')
                elif re.match(r'.*meson.*', name) or re.match(r'.*Makefile.*', name):
                        copyright = find_copyright_pound(name)
                elif re.match(r'.*\.[123]$', name):
                        copyright = find_copyright_troff(name)
                if not copyright:
                        m = magic.from_file(name)
                        if m is None:
                                copyright = 'unknown file type'
                        else:
                                if re.search(r'troff', m):
                                        copyright = find_copyright_troff(name)
                                elif re.search(r'C source', m):
                                        copyright = find_copyright_source(name, 'text/x-c')
                                elif (re.search(r'POSIX shell script', m) or
                                      re.search(r'Bourne-Again shell script', m) or
                                      re.search(r'Python script', m) or
                                      re.search(r'Perl script', m) or
                                      starts_with(r'#', name)):
                                        copyright = find_copyright_pound(name)
                if not copyright:

                        # Skip very short files without a copyright

                        if num_lines(name) < 10:
                                continue

                        # Skip generated files

                        if (file_contains(r'generated automatically', name) or
                            file_contains(r'automatically generated', name)):
                                continue

                        copyright = ''

                i = pack_copyright(copyright)
                if i not in copyrights:
                        copyrights[i] = copyright
                        copyright_users[i] = [name]
                else:
                        copyright_users[i] += [name]
        
        license_map = {}
        licenses = {}
        license_ids = {}
        license_name = {}
        holders = {}
        copyright_map = {}
        
        for i in copyrights:
                holder = get_holder(copyrights[i])
                license = get_license(copyrights[i])
                license_key = pack_copyright(license)
                type = get_license_type(license)
                if license_key in license_name:
                        name = license_name[license_key]
                else:
                        if type in license_ids:
                                id = license_ids[type]
                        else:
                                id = 1
                        license_ids[type] = id + 1
                        name = "%s-%d" % (type, id)
                        licenses[name] = license
                        holders[name] = holder
                        license_name[license_key] = name
                if name in copyright_map:
                        copyright_map[name] += [i]
                else:
                        copyright_map[name] = [i]

        for type in sorted(license_ids.keys()):
                for id in range(1,license_ids[type]):

                        name = "%s-%d" % (type, id)

                        for i in copyright_map[name]:
                                holder = get_holder(copyrights[i])
                                license = get_license(copyrights[i])

                                print("Files:", end='')
                                for file in copyright_users[i]:
                                        print(" %s" % file)
                                print("Copyright:", end='')
                                done = False
                                for line in holder.splitlines():
                                        for pat in (r'(.*)copyright(.*)',
                                                    r'(.*)©(.*)', r'(.*)\(c\)(.*)',
                                                    r'(.*) by (.*)',
                                                    r'(.*)all rights reserved\.?(.*)'):
                                                m = re.match(pat, line, re.I)
                                                if m:
                                                        line = m.group(1).strip() + ' ' + m.group(2).strip()
                                        line = line.strip()
                                        if re.match(r'(19[89][0-9]|2[0-2][0-9][0-9]).*', line):
                                                if done:
                                                        print()
                                        print(' %s' % line, end='')
                                        done = True
                                print()
                                print("License: %s" % name)
                                print()

        started = False
        for type in sorted(license_ids.keys()):
                for id in range(1,license_ids[type]):
                        if started:
                                print()
                        started = True
                        name = "%s-%d" % (type, id)
                        print("License: %s" % name)
                        for line in licenses[name].splitlines():
                                if len(line) == 0:
                                        line = '.'
                                print(" %s" % line)
main()