lang_sync

#!/usr/bin/env python3

import base64
import codecs
import csv
import getopt
import getpass
import io
import os
import re
import sys

import urllib.request

KNOWN_PROJECT_TYPES = ["game-script", "newgrf", "openttd"]

PROJECT_TYPES_TEXT = ['"{}"'.format(pt) for pt in KNOWN_PROJECT_TYPES]
PROJECT_TYPES_TEXT = ", ".join(PROJECT_TYPES_TEXT[:-1]) + " and " + PROJECT_TYPES_TEXT[-1]

hex_pat = re.compile("0x([0-9A-Fa-f][0-9A-Fa-f]?)$")

# Configuration.
project_pat = re.compile("[-A-Za-z0-9_]+$")


def check_project_name(name):
    """
    Check whether L{name} can be a valid project name.

    @param name: Name to verify.
    @type  name: C{str}

    @return: Whether the given name is valid.
    @rtype:  C{bool}
    """
    return project_pat.match(name.strip())


def check_human_name(name):
    """
    Check whether L{name} can be a valid human project name.

    @param name: Name to verify.
    @type  name: C{str}

    @return: Whether the given name is valid.
    @rtype:  C{bool}
    """
    return len(name.strip()) != 0


def check_project_url(url):
    """
    Check whether the given L{url} is valid.

    @param url: Url of the project, may be empty.
    @type  url: C{str}

    @return: Whether the provided url is valid.
    @rtype:  C{bool}
    """
    if url == "":
        return True
    if url.startswith("http://"):
        return True
    if url.startswith("https://"):
        return True
    return False


class UserConfig:
    """
    Configuration and actions to do by the program.

    @ivar filename: Name of the file containing authentication information, if available.
    @type filename: C{str} or C{None}

    @ivar user: User name to use, if available.
    @type user: C{str} or C{None}

    @ivar passwd: Password to use, if available.
    @type passwd: C{str} or C{None}

    @ivar interactive: Can the program ask for the password interactively? (Default: C{True}).
    @type interactive: C{bool}

    @ivar base_url: Base url of the authentication realm (the scheme + host), if available.
    @type base_url: C{str} or C{None}

    @ivar project: Project identifier of Eints, if available.
    @type project: C{str} or C{None}

    @ivar project_type: Type of the project.
    @type project_type: C{str}

    @ivar project_desc: Project description, if available.
    @type project_desc: C{str} or C{None}

    @ivar project_url: Project URL, may be empty.
    @type project_url: C{str}

    @ivar lang_dir: Directory containing the language files relative to the working directory.
    @type lang_dir: C{str}

    @ivar unstable_lang_dir: Directory containing the unstable language files relative to the working directory.
    @type unstable_lang_dir: C{str}

    @ivar lang_ext: Extension used by language files.
    @type lang_ext: C{str}

    @ivar base_lang: Isocode or grf language id of the base language, if available.
    @type base_lang: C{str} or C{None}

    @ivar credits_file: If available, path to a file for writing credits for changes in the language files.
    @type credits_file: C{str} or C{None}

    @ivar verbose: Whether to be verbose in what actions the program performs.
    @type verbose: C{bool}

    @ivar no_write: Whether to skip writing data onto the disk.
    @type no_write: C{bool}

    @ivar operations: Upload and download actions to perform.
    @type operations: C{list} of C{str}
    """

    def __init__(self):
        self.filename = None
        self.user = None
        self.passwd = None
        self.interactive = True
        self.base_url = None
        self.base_lang = None
        self.credits_file = None
        self.project = None
        self.project_type = "**unset**"
        self.project_desc = None
        self.project_url = ""
        self.lang_dir = "lang"
        self.unstable_lang_dir = None
        self.lang_ext = ".lng"
        self.verbose = False
        self.no_write = False
        self.operations = []

    def check(self):
        """
        Check whether enough information was provided to the program. If not,
        fail with an error message.
        """
        if self.base_url is None:
            print("Error: Missing base url to the Eints server (add it with --base-url=URL).")
            sys.exit(1)
        if self.project is None:
            print("Error: Missing Eints project identifier (add it with --project=PROJ_ID).")
            sys.exit(1)
        if not check_project_name(self.project):
            print('Error: Project identifier "{}" is not valid.'.format(self.project))
            sys.exit(1)
        if self.project_desc is not None and not check_human_name(self.project_desc):
            print('Error: Project description "{}" is not valid.'.format(self.project_desc))
            sys.exit(1)
        if not check_project_url(self.project_url):
            print('Error: Project url "{}" is not valid.'.format(self.project_url))
            sys.exit(1)
        if len(self.operations) == 0:
            print("Warning: No operations to perform, finishing now.")
            sys.exit(0)


user_cfg = None  #: Global user configuration.


# Network access code.
def setup_auth():
    """
    Setup the authentication for connecting to Eints.
    """
    global user_cfg

    user, passwd = None, None

    # Read the file if provided.
    if user_cfg.filename is not None:
        if not os.path.isfile(user_cfg.filename):
            print('Error: File "{}" does not exist or is not a file.'.format(user_cfg.filename))
            sys.exit(1)
        with open(user_cfg.filename, "r", encoding="utf-8") as handle:
            for line in handle:
                line = line.strip()

                # Check for a well-known prefix.
                if line.startswith("username"):
                    prefix = "u"
                    line = line[8:].lstrip()
                elif line.startswith("user"):
                    prefix = "u"
                    line = line[4:].lstrip()
                elif line.startswith("password"):
                    prefix = "p"
                    line = line[8:].lstrip()
                elif line.startswith("pwd"):
                    prefix = "p"
                    line = line[3:].lstrip()
                else:
                    continue  # No match for user or password

                # Silently drop the : or = delimiter.
                if line[0] in ":=":
                    line = line[1:].lstrip()

                if len(line) > 0 and line[0] in ('"', "'") and line[0] == line[-1]:
                    line = line[1:-1]

                if prefix == "u":
                    user = line
                else:
                    passwd = line

        # Close of the file is implicit.

    if user is None:
        user = user_cfg.user
        if user is None:
            user = getpass.getuser()

    if passwd is None:
        passwd = user_cfg.passwd
        if passwd is None and user_cfg.interactive:
            passwd = getpass.getpass()

    if user is not None and passwd is not None:

        class ForceHTTPBasicAuthHandler(urllib.request.AbstractBasicAuthHandler, urllib.request.BaseHandler):
            auth_header = "Authorization"

            # On every request, add an Authorization header
            def http_request(self, req):
                user, passwd = self.passwd.find_user_password(None, req.full_url)
                credentials = "{0}:{1}".format(user, passwd).encode()
                auth_str = base64.standard_b64encode(credentials).decode()
                req.add_unredirected_header("Authorization", "Basic {}".format(auth_str.strip()))
                return req

            https_request = http_request

        # If both a user and a password, we can usefully setup authentication.
        pwd_manager = urllib.request.HTTPPasswordMgr()
        pwd_manager.add_password(None, user_cfg.base_url, user, passwd)
        auth_handler = ForceHTTPBasicAuthHandler(pwd_manager)
        opener = urllib.request.build_opener(auth_handler)
        urllib.request.install_opener(opener)

        if user_cfg.verbose:
            print("Installed authentication.")
            print()
        return

    if user_cfg.verbose:
        text = "Skipped installing authentication"
        if user is None:
            text = text + ", user was not set"
        if passwd is None:
            text = text + ", password was not set"
        text = text + "."
        print(text)
        print()


def build_url(suffix):
    """
    Extend the base url with the provided suffix.

    @param suffix: Path to retrieve from, at L{UserConfig.base_url}
    @type  suffix: C{str}
    """
    global user_cfg

    assert suffix[0] == "/"
    if user_cfg.base_url[-1] == "/":
        return user_cfg.base_url + suffix[1:]
    else:
        return user_cfg.base_url + suffix


def get_data(url):
    """
    Read and return the data read from the L{url}.

    @param url: Url for the file to retrieve.
    @type  url: C{str}

    @return: Data returned by the L{url} request, if all went well.
    @rtype:  C{str} or C{None}
    """
    try:
        with urllib.request.urlopen(url) as resp:
            if resp.status != 200:
                return None

            ctype = resp.getheader("Content-Type")
            if ctype is None or ctype != "text/plain; charset=UTF-8":
                return None

            data = resp.read().decode("utf-8")
            return data
    except Exception:
        print("Error: Failing URL: {}".format(url))
        return None


def post_form(url, form_data):
    """
    Post the form.

    @param url: Url for the file to retrieve.
    @type  url: C{str}

    @param form_data: Form data.
    @type  form_data: C{dict} of C{str} to C{str}

    @return: Whether the from was posted correctly.
    @rtype:  C{bool}
    """
    headers = {"Content-Type": " application/x-www-form-urlencoded; charset=utf-8"}
    form = urllib.parse.urlencode(form_data).encode("utf-8")
    req = urllib.request.Request(url, form, headers)

    # Perform the request, and read the returned data.
    with urllib.request.urlopen(req) as resp:
        if resp.status != 200:
            return False

        resp.read()
        return True


def post_form_upload(url, form_data, form_fentry, form_fname, fdata):
    """
    Post the form with the uploaded files.

    @param form_data: Form data.
    @type  form_data: C{dict} of C{str} to C{str}

    @param form_fentry: Form field of the file entry.
    @type  form_fentry: C{str}

    @param form_fname: Name of the file (mostly for extension).
    @type  form_fname: C{str}

    @param fdata: File data.
    @type  files: C{str}

    @return: Whether posting was successful.
    @rtype:  C{bool}
    """
    boundary = "TheBoundaryOfFormDataEntries"

    data = []
    for field, value in form_data.items():
        data.append("--" + boundary)
        data.append('Content-Disposition: form-data; name="{}"'.format(field))
        data.append("")
        data.append(value)

    data.append("--" + boundary)
    data.append('Content-Disposition: form-data; name="{}"; filename="{}"'.format(form_fentry, form_fname))
    data.append("Content-Type: text/plain")
    data.append("")
    data.append(fdata)
    if fdata[-1] != "\n":
        data.append("")  # Add a \n ourselves

    data.append("--" + boundary + "--\n")

    # Send the data.
    headers = {"Content-Type": " multipart/form-data; charset=utf-8; boundary={}".format(boundary)}
    data = "\n".join(data).encode("utf-8")
    req = urllib.request.Request(url, data, headers)

    # Perform the request, and return the read data.
    with urllib.request.urlopen(req) as resp:
        if resp.status != 200 or resp.geturl().find("Successfully") < 0:
            return False

        resp.read()
        return True


# Remote language files.
class EintsLanguageInfo:
    """
    Language information received from Eints.

    @ivar isocode: Isocode of the of the language.
    @type isocode: C{str}

    @ivar parent_lang: Isocode of the language followed by this language, if it exists.
    @type parent_lang: C{str} or C{None} if it is the root language

    @ivar last_change: Last change of the language, if it exists.
    @type last_change: C{None} or C{str}
    """

    def __init__(self, isocode, name, parent_lang, last_change):
        self.isocode = isocode
        self.name = name
        self.parent_lang = parent_lang
        self.last_change = last_change


eints_files_cache = None


def get_eints_languages():
    """
    Get the list of languages available in Eints.

    @return: Available language information for the project as returned by Eints.
    @rtype:  C{list} of L{EintsLanguageInfo}
    """
    global user_cfg, eints_files_cache

    if eints_files_cache is not None:
        return eints_files_cache

    if user_cfg.verbose:
        print("Getting available language files from Eints.")
    result = []
    data = get_data(build_url("/download-list/" + user_cfg.project))
    if data is None:
        print("Error: Could not download the list of project languages from Eints.")
        sys.exit(1)

    reader = csv.DictReader(io.StringIO(data), dialect="unix")
    for line in reader:
        isocode = line.get("isocode")
        if isocode is None or len(isocode) == 0:
            continue
        name = line.get("name", isocode)

        plang = line.get("base_isocode")
        if plang is None:
            continue
        if len(plang) == 0:
            plang = None

        # last_change = line.get("changetime")
        # XXX Parse and validate the last change date

        el = EintsLanguageInfo(isocode, name, plang, None)
        result.append(el)

    eints_files_cache = result

    if user_cfg.verbose:
        if len(eints_files_cache) == 0:
            print("No languages found in Eints.")
        else:
            print("Found remote language files:")
            for el in eints_files_cache:
                if el.parent_lang is None:
                    kind = "(base language)"
                else:
                    kind = "(translation of {})".format(el.parent_lang)
                print("\tisocode: {} {}".format(el.isocode, kind))
            print()

    return eints_files_cache


# Local language files.
class LocalLanguageInfo:
    """
    Information about a local language file.

    @ivar path_name: The path to the file.
    @type path_name: C{str}

    @ivar isocode: Isocode of the language,
    @type isocode: C{str}
    """

    def __init__(self, path_name, isocode):
        self.path_name = path_name
        self.isocode = isocode


utf8_bom = codecs.BOM_UTF8.decode("utf-8")
local_files_cache = None


def get_local_languages():
    """
    Get the list of available languages at the local file system.

    @return: Found language files.
    @rtype:  C{list} of L{LocalLanguageInfo}
    """
    global user_cfg, local_files_cache

    if local_files_cache is not None:
        return local_files_cache

    if user_cfg.verbose:
        print("Getting available language files from the local file system.")
    result = []
    grflangid_pattern = re.compile("##grflangid\\s+0[xX]([0-9A-Fa-f][0-9A-Fa-f]?)\\s*$")
    if not os.path.isdir(user_cfg.lang_dir):
        print('Error: "{}" is not a directory. (Is the --lang-dir option correct?)'.format(user_cfg.lang_dir))
        sys.exit(1)
    if user_cfg.unstable_lang_dir is not None and not os.path.isdir(user_cfg.lang_dir):
        print(
            'Error: "{}" is not a directory. (Is the --unstable-lang-dir option correct?)'.format(
                user_cfg.unstable_lang_dir
            )
        )
        sys.exit(1)

    searchpaths = [user_cfg.lang_dir]
    if user_cfg.unstable_lang_dir is not None:
        searchpaths.append(user_cfg.unstable_lang_dir)

    for spath in searchpaths:
        for fname in os.listdir(spath):
            if not fname.endswith(user_cfg.lang_ext):
                continue
            path = os.path.join(spath, fname)
            if user_cfg.verbose:
                print('File "{}" matches extension'.format(path))

            # Try reading the grflangid from the file.
            grflangid = None
            with open(path, "r", encoding="utf-8") as handle:
                for line in handle:
                    if line.startswith(utf8_bom):
                        line = line[len(utf8_bom) :]
                    m = grflangid_pattern.match(line)
                    if m:
                        grflangid = int(m.group(1), 16)
                        break

            # If not found, try to derive it from the filename.
            if grflangid is None:
                ldata = get_languagedata_by_filename(fname[: -len(user_cfg.lang_ext)])
            else:
                ldata = get_languagedata_by_grflangid(grflangid)

            if ldata is not None:
                result.append(LocalLanguageInfo(path, ldata.isocode))
                continue

            if user_cfg.verbose:
                print('File "{}" matches extension, but cannot derive a language from it.'.format(path))

    local_files_cache = result

    if user_cfg.verbose:
        if len(local_files_cache) == 0:
            print("No languages found locally.")
        else:
            print("Found local language files:")
            for ll in local_files_cache:
                print("\tisocode: {}, path: {}".format(ll.isocode, ll.path_name))
            print()

    return local_files_cache


# Upload and download operations.
def perform_operation(op_type, lang_type, credits_handle):
    """
    Perform a synchronization operation.

    @param op_type: Type of operation, either C{"upload"}, C{"download"}, or C{"create"}.
    @type  op_type: C{str}

    @param lang_type: Type of languages involved, either C{"base"}, C{"translations"}, or C{"project"}.
    @type  lang_type: C{str}

    @param credits_handle: If available, output stream for writing credits of language file changes.
    @type  credits_handle: C{file} or C{None}
    """
    global user_cfg

    if op_type == "create":
        assert lang_type == "project"
        # Project creation operation.
        if user_cfg.project_desc is None:
            print("Error: Missing project description (add it with --project-desc).")
            sys.exit(1)

        perform_create_project()
        return

    # Upload/download operations below.
    ll_files = get_local_languages()
    el_files = get_eints_languages()

    # Find out what the base language is.
    blng = user_cfg.base_lang
    if blng:
        if hex_pat.match(blng):
            blng = get_languagedata_by_grflangid(int(blng, 16))
        else:
            blng = get_languagedata_by_isocode(blng)

    for el in el_files:
        if el.parent_lang is None:
            if blng is not None and blng.isocode != el.isocode:
                text = "Error: Base languages do not match, --base-lang claims {} while remote claims {}"
                print(text.format(blng.isocode, el.isocode))
                sys.exit(1)
            blng = get_languagedata_by_isocode(el.isocode)
            break

    if blng is None:
        print("Error: No base language found (neither from Eints nor given with --base-lang)")
        sys.exit(1)

    # Filter languages.
    path_base = None
    if lang_type == "base":
        ll_files = [ll for ll in ll_files if ll.isocode == blng.isocode]
        el_files = [el for el in el_files if el.isocode == blng.isocode]
    else:
        path_base = next((ll.path_name for ll in ll_files if ll.isocode == blng.isocode), None)
        ll_files = [ll for ll in ll_files if ll.isocode != blng.isocode]
        el_files = [el for el in el_files if el.isocode != blng.isocode]

    if op_type == "upload":
        perform_upload(ll_files, lang_type == "base")
    else:
        perform_download(ll_files, el_files, credits_handle, path_base)


def perform_create_project():
    """
    Create a new project in Eints.
    """
    global user_cfg

    assert user_cfg.project_desc is not None

    if user_cfg.project_type not in KNOWN_PROJECT_TYPES:
        print('Error: Project type "{}" is not known.'.format(user_cfg.project_type))
        sys.exit(1)

    fields = {"humanname": user_cfg.project_desc, "url": user_cfg.project_url}
    url = build_url("/makeproject/" + user_cfg.project_type + "/" + user_cfg.project)
    if user_cfg.no_write:
        print('Skipped creation of project "{}".'.format(user_cfg.project))
    else:
        if not post_form(url, fields):
            print('Project creation of "{}" failed.'.format(user_cfg.project))
            sys.exit(1)
        if user_cfg.verbose:
            print('Created project "{}".'.format(user_cfg.project))


def perform_upload(ll_files, is_base):
    """
    Perform upload of local files to Eints.

    @param ll_files: Local files to upload.
    @type  ll_files: C{list} of L{LocalLanguageInfo}

    @param is_base: Whether uploading the base language.
    @type  is_base: C{bool}
    """
    if is_base:
        blng_value = "on"
    else:
        blng_value = ""

    for ll in ll_files:
        with open(ll.path_name, "r", encoding="utf-8") as handle:
            data = handle.read()

        data = data.rstrip()
        if len(data) == 0:
            print('Warning: Skipping empty "{}".'.format(ll.path_name))
            continue

        fields = {"override": "", "base_language": blng_value}
        fname = os.path.basename(ll.path_name)
        if user_cfg.no_write:
            if user_cfg.verbose:
                print('Skipped upload of "{}"'.format(fname))
            ok = True
        else:
            if user_cfg.verbose:
                print('Uploaded "{}" to Eints'.format(fname))

            url = build_url("/upload/" + user_cfg.project + "/" + ll.isocode)
            ok = post_form_upload(url, fields, "langfile", fname, data)

        if not ok:
            print('Error: Failed to upload "{}".'.format(ll.path_name))


class LanguageData:
    """
    Language information.

    @ivar grflangid: NewGRF language ID.
    @type grflangid: C{int}

    @ivar isocode: ISO code of the language.
    @type isocode: C{str}

    @ivar filename: Base filename of the language.
    @type filename: C{str}

    @ivar is_stable: Whether the language is considered stable.
    @type is_stable: C{bool}
    """

    def __init__(self, grflangid, isocode, filename, is_stable):
        self.grflangid = grflangid
        self.isocode = isocode
        self.filename = filename
        self.is_stable = is_stable


language_table = []

filename_pattern = re.compile("[0-9a-zA-Z_]+$")


def get_language_data():
    data = get_data(build_url("/language-list"))
    if data is None:
        print("Error: Could not download the list of language definitions from Eints.")
        sys.exit(1)

    reader = csv.DictReader(io.StringIO(data), dialect="unix")
    for line in reader:
        filename = line.get("filename")
        if filename is None or not filename_pattern.match(filename):
            continue

        isocode = line.get("isocode")
        if isocode is None or len(isocode) == 0:
            continue

        grflangid = line.get("grflangid")
        if grflangid is None or not hex_pat.match(grflangid):
            continue
        grflangid = int(grflangid, 16)

        is_stable = line.get("is_stable")
        if is_stable is None or is_stable not in ["True", "False"]:
            continue
        is_stable = is_stable == "True"

        language_table.append(LanguageData(grflangid, isocode, filename, is_stable))


def get_languagedata_by_isocode(isocode):
    """
    Retrieve the language data from the table that matches the L{grflangid} parameter.

    @param isocode: Isocode of the language.
    @type  isocode: C{str}

    @return: Language data, if available.
    @rtype:  L{LanguageData} or C{None}
    """
    if len(language_table) == 0:
        get_language_data()

    for ld in language_table:
        if ld.isocode == isocode:
            return ld
    return None


def get_languagedata_by_grflangid(grflangid):
    """
    Retrieve the language data from the table that matches the L{grflangid} parameter.

    @param grflangid: NewGRF language ID.
    @type  grflangid: C{int}

    @return: Language data, if available.
    @rtype:  L{LanguageData} or C{None}
    """
    if len(language_table) == 0:
        get_language_data()

    for ld in language_table:
        if ld.grflangid == grflangid:
            return ld
    return None


def get_languagedata_by_filename(filename):
    """
    Retrieve the language data from the table that matches the L{filename} parameter.

    @param filename: Filename of the language file.
    @type  filename: C{str}

    @return: Language data, if available.
    @rtype:  L{LanguageData} or C{None}
    """
    if len(language_table) == 0:
        get_language_data()

    for ld in language_table:
        if ld.filename == filename:
            return ld
    return None


string_pat = re.compile("^([A-Za-z_0-9]+)(\\.[A-Za-z0-9]+)?[ \\t]*:(.*)$")


def parse_string(line):
    """
    Parse a string, and return its name and it text if parsing succeeds.

    @param line: Line of text containing the string.
    @type  line: C{str}

    @return: A tuple (name, text) or C{None}
    @rtype:  (C{str}, C{str}) or C{None}
    """
    m = string_pat.match(line)
    if not m:
        return None

    if m.group(2) is None:
        return m.group(1), m.group(3).rstrip()
    else:
        return m.group(1) + m.group(2), m.group(3).rstrip()


def load_local_strings(path):
    """
    Load the strings of a local file, and store them in a set for fast comparing.

    @param path: Path of the file to load.
    @type  path: C{str}

    @return: Strings of the file.
    @rtype:  C{set} of C{str}
    """
    strings = set()
    with open(path, "r", encoding="utf-8") as handle:
        for line in handle:
            value = parse_string(line)
            if value is not None:
                strings.add(value)

    return strings


def parse_annotated_langfile(data):
    """
    Parse an annotated language file loaded from eints.

    @param data: Downloaded data.
    @type  data: C{str}

    @return: Parsed annotated language file lines, a sequence of tuples (type, <data>).
             The following parsed lines are returned:
             - ('text', <text-to-copy>)
             - ('string', <string-line>, <credits>), where <credits> may be C{None}
    @rtype:  C{list} of (C{str}, C{str},  [C{str}])
    """
    parsed = []
    string_text = None
    for line in data.splitlines():
        if string_text is not None:
            if line.startswith("credits:"):
                parsed.append(("string", string_text, line[8:]))
                string_text = None
                continue
            parsed.append(("string", string_text, None))
            string_text = None

        if line.startswith("string:"):
            string_text = line[7:]  # Temporary store the string, to check for credits at the next line.
            continue

        if line.startswith("literal:"):
            parsed.append(("text", line[8:]))
            continue

        if line.startswith("pragma:"):
            parsed.append(("text", line[7:]))
            continue

        if line.startswith("gender:") or line.startswith("plural:"):
            parsed.append(("text", line[7:]))
            continue

        if line.startswith("case:"):
            parsed.append(("text", line[5:]))
            continue

        if line.startswith("grflangid:"):
            parsed.append(("text", line[10:]))
            continue

    if string_text is not None:
        parsed.append(("string", string_text, None))

    return parsed


def collect_credits(strings, parsed):
    """
    Find changed language file strings, and collect credits for who changed them.

    @param strings: Strings of the local file.
    @type  strings: C{list} of (C{str}, C{str})

    @param parsed: Parsed annotated language file.
    @type  parsed: C{list} of (C{str}, C{str}, C{str} or C{None})
    """
    credits = {}
    for p in parsed:
        if p[0] != "string":
            continue  # Not a string.
        value = parse_string(p[1])
        if value is None or p[2] is None or value in strings:
            continue

        n = credits.get(p[2])
        if n is None:
            credits[p[2]] = 1
        else:
            credits[p[2]] = n + 1

    persons = []
    for user, count in credits.items():
        if count == 1:
            persons.append((1, "1 change by " + user))
        else:
            persons.append((count, str(count) + " changes by " + user))

    return ", ".join(p[1] for p in sorted(persons, reverse=True))


def detect_eolstyle(path):
    """
    Detect eolstyle of a file.

    @param path: Path of the file to rad.
    @type  path: C{str}

    @return: EOL style. '\n', '\r' or '\r\n' or None for unknown (no linebreaks at all).
    @rtype:  C{str}
    """
    with open(path, "r", encoding="utf-8") as f:
        f.readline()
        if type(f.newlines) == tuple:
            return f.newlines[0]
        else:
            return f.newlines


def write_parsed_lines(parsed, path, eolstyle):
    """
    Write a language file previously parsed into lines by L{parse_annotated_langfile} to the local disc.

    @param parsed: Parsed lines of the annotated language file.
    @type  parsed: C{list} of (C{str}, C{str},  C{str} or C{None})

    @param path: Path of the file to write.
    @type  path: C{str}

    @param eolstyle: EOL style. '\n', '\r' or '\r\n' or None for native.
    @type  eolstyle: C{str} or C{None}
    """
    with open(path, "w", encoding="utf-8", newline=eolstyle) as handle:
        handle.write("\n".join(p[1] for p in parsed))
        handle.write("\n")


def perform_download(ll_files, el_files, credits_handle, path_base):
    """
    Download Eints files to local files.

    @param ll_files: Local files to overwrite.
    @type  ll_files: C{list} of L{LocalLanguageInfo}

    @param el_files: Eints files to download.
    @type  el_files: C{list} of L{EintsLanguageInfo}

    @param credits_handle: If available, output stream for writing credits of language file changes.
    @type  credits_handle: C{file} or C{None}

    @param path_base: Local baselang file.
    @type  path_base: C{str} or C{None}
    """
    local_files = dict((ll.isocode, ll) for ll in ll_files)  # Mapping of isocode to local info.

    eolstyle_baselang = None
    if path_base is not None:
        eolstyle_baselang = detect_eolstyle(path_base)

    for el in el_files:
        url = build_url("/annotate/" + user_cfg.project + "/" + el.isocode)
        data = get_data(url)
        if data is None:
            print("Error: Could not download the {} language from Eints.".format(el.isocode))
            continue
        ll = local_files.get(el.isocode)
        eolstyle = None
        if ll is not None:
            path = ll.path_name
            eolstyle = detect_eolstyle(path)
            action = "Wrote"
            if credits_handle is None:
                current_strings = None  # Causes a crash if you try to use it.
            else:
                current_strings = load_local_strings(path)
        else:
            ldata = get_languagedata_by_isocode(el.isocode)
            if ldata is None:
                path = el.isocode
            else:
                path = ldata.filename

            if ldata.is_stable:
                path = user_cfg.lang_dir + os.sep + path + user_cfg.lang_ext
            elif user_cfg.unstable_lang_dir is not None:
                path = user_cfg.unstable_lang_dir + os.sep + path + user_cfg.lang_ext
            else:
                continue

            action = "Created"
            current_strings = set()

        if eolstyle is None:
            eolstyle = eolstyle_baselang

        if user_cfg.no_write:
            action = "Skipped " + action.lower()
        else:
            parsed = parse_annotated_langfile(data)
            if credits_handle is not None:
                credits = collect_credits(current_strings, parsed)
                if len(credits) > 0:
                    credits_handle.write(el.name.lower() + ": " + credits + "\n")
                    credits_handle.flush()  # Flush it to disk, so it is preserved on crash.
            write_parsed_lines(parsed, path, eolstyle)

        if user_cfg.verbose:
            print('{} language file "{}"'.format(action, path))


USAGE_TEXT = (
    """\
lang_sync -- Synchronize language files between the project and Eints.

lang_sync <options> <operations>


with <options>:


General options:

--help
-h
    Get this help text.

--verbose
    Show what the program is doing

--no-write
    Don't make actual changes to any file




Getting the Eints user name and password information:

--user-password-file=FILE
    If given, FILE contains lines of the form "username: USERNAME" and/or
    "password: PASSWORD".

--user=USERNAME
--password=PASSWORD
    If no file is given, or it did not contain all information, these options
    may be used to specify the user name and password at the command line.

    At a multi-user system this is not secure, and should be avoided.

--not-interactive
    The last option to get the user name and password is using the standard
    getpass module. This will ask for the password from the command-line.
    This option prevents use of this fallback method, so lang_sync will not
    become stuck at waiting for user input.

--base-url=BASE_URL
    The scheme and host name part of the Eints server urls. For example,
    "https://eints.example.com/".




Getting project information:

--project=PROJECT_NAME
    Name of the project in Eints (may contain letters, digits, and dashes).

--project-type=PROJECT_TYPE
    Type of the project. Accepted values are """
    + PROJECT_TYPES_TEXT
    + "."
    + """

--project-desc=PROJECT_DESCRIPTION
    For project creation, a longer title of the project (may contain letters,
    digits, dashes and spaces).

--project-url=PROJECT_URL
    For project creation, an optional website URL.

--lang-dir=LANG_DIR
    Path of the directory containing the language files at the local disc.
    (Default: "lang")

--unstable-lang-dir=UNSTABLE_LANG_DIR
    Path of the directory containing the unstable language files at the local disc.
    (Default: None, no unstable languages are downloaded)

--lang-file-ext=LANG_EXT
    Filename suffix used by the language files. (Default: ".lng")

--base-language=BASE_LANG
    In case Eints does not know the base language (mostly at initial import),
    this option can be used to specify it manually. It can either be the isocode,
    or the ##grflangid (a string of the form "0x[HEX][HEX]").

--credits=FNAME
    Credits for translation string changes are written in the provided FNAME.



and <operations>:

download-base
    Download the base language file from Eints.

download-translations
    Download all the translations from Eints.

upload-base
    Upload the base language file to Eints.

upload-translations
    Upload all the translation files to Eints.

create-project
    Create a new project in Eints.

You can give several operations at the command line, order is significant.
"""
)


def run():
    """
    Run the program (it was started from the command line).
    """
    global user_cfg

    long_opts = [
        "user-password-file=",
        "user=",
        "password=",
        "not-interactive",
        "base-url=",
        "project=",
        "project-type=",
        "project-desc=",
        "project-url=",
        "lang-dir=",
        "unstable-lang-dir=",
        "base-language=",
        "lang-file-ext=",
        "credits=",
        "help",
        "verbose",
        "no-write",
    ]

    try:
        opts, args = getopt.getopt(sys.argv[1:], "h", long_opts)
    except getopt.GetoptError as err:
        print("lang_sync: " + str(err) + ' (try "lang_sync --help")')
        sys.exit(2)

    user_cfg = UserConfig()
    for opt, val in opts:
        if opt in ("--help", "-h"):
            print(USAGE_TEXT)
            sys.exit(0)

        if opt == "--verbose":
            user_cfg.verbose = True
            continue
        if opt == "--no-write":
            user_cfg.no_write = True
            continue
        if opt == "--base-language":
            user_cfg.base_lang = val
            continue
        if opt == "--lang-dir":
            user_cfg.lang_dir = val
            continue
        if opt == "--unstable-lang-dir":
            user_cfg.unstable_lang_dir = val
            continue
        if opt == "--lang-file-ext":
            user_cfg.lang_ext = val
            continue
        if opt == "--base-url":
            user_cfg.base_url = val
            continue
        if opt == "--credits":
            user_cfg.credits_file = val
            continue
        if opt == "--project":
            user_cfg.project = val
            continue
        if opt == "--project-type":
            user_cfg.project_type = val
            continue
        if opt == "--project-desc":
            user_cfg.project_desc = val
            continue
        if opt == "--project-url":
            user_cfg.project_url = val
            continue
        if opt == "--user-password-file":
            user_cfg.filename = val
            continue
        if opt == "--user":
            user_cfg.user = val
            continue
        if opt == "--password":
            user_cfg.passwd = val
            continue
        if opt == "--not-interactive":
            user_cfg.interactive = False
            continue

        raise ValueError("Unknown option {} encountered.".format(opt))

    user_cfg.operations = []
    done_down_base = False
    done_down_trans = False
    for arg in args:
        if arg == "download-base":
            done_down_base = True
            user_cfg.operations.append(("download", "base"))
        elif arg == "download-translations":
            done_down_trans = True
            user_cfg.operations.append(("download", "translations"))
        elif arg == "upload-base":
            if done_down_base:
                print("Warning: Uploading the base language after downloading it does not copy local changes.")
            user_cfg.operations.append(("upload", "base"))
        elif arg == "upload-translations":
            if done_down_trans:
                print("Warning: Uploading the translations after downloading them does not copy local changes.")
            user_cfg.operations.append(("upload", "translations"))
        elif arg == "create-project":
            user_cfg.operations.append(("create", "project"))
        else:
            text = (
                'Argument "{}" is not a valid synchronization operation.  '
                "Try 'lang_sync --help' for more information."
            )
            text = text.format(arg)
            print(text)
            sys.exit(1)
    user_cfg.check()

    setup_auth()

    credits_handle = None
    try:
        if user_cfg.credits_file:
            credits_handle = open(user_cfg.credits_file, "w", encoding="utf-8")

        for op_type, lng_type in user_cfg.operations:
            if user_cfg.verbose:
                print('Performing "{}-{}"'.format(op_type, lng_type))
            perform_operation(op_type, lng_type, credits_handle)

    finally:
        if credits_handle:
            credits_handle.close()

    sys.exit(0)


if __name__ == "__main__":
    run()