From a906bd7735d7a31da09d282083de3fb82c1c030e Mon Sep 17 00:00:00 2001 From: Adriaan de Groot Date: Tue, 19 Jun 2018 08:29:17 -0400 Subject: [PATCH 1/7] Tidy up the locale.gen handling. There is no real format in this file: just lines which might be comments, might be human-readable-text, or might be example locales. Instead of modifying any comment-lines, write enabled locales at the bottom of the file, while matching very loosely with the comment lines from the file. FIXES #940 (insofar as that is fixable) From 863d00f40c00fa0b551fd96f6a5302213c2a76bc Mon Sep 17 00:00:00 2001 From: Adriaan de Groot Date: Tue, 19 Jun 2018 08:42:25 -0400 Subject: [PATCH 2/7] [localecfg] Document purpose of this module --- src/modules/localecfg/main.py | 1 + src/modules/localecfg/module.desc | 3 +++ 2 files changed, 4 insertions(+) diff --git a/src/modules/localecfg/main.py b/src/modules/localecfg/main.py index d44d7da2b3..55c704fae9 100644 --- a/src/modules/localecfg/main.py +++ b/src/modules/localecfg/main.py @@ -7,6 +7,7 @@ # Copyright 2015, Philip Müller # Copyright 2016, Teo Mrnjavac # Copyright 2018, AlmAck +# Copyright 2018, Adriaan de Groot # # Calamares is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/src/modules/localecfg/module.desc b/src/modules/localecfg/module.desc index 89baab7add..8154805620 100644 --- a/src/modules/localecfg/module.desc +++ b/src/modules/localecfg/module.desc @@ -1,3 +1,6 @@ +# Enable the configured locales (those set by the user on the +# user page) in /etc/locale.gen, if they are available in the +# target system. --- type: "job" name: "localecfg" From 7498629b5f71e189d63468ac6e49ff35080f2ed2 Mon Sep 17 00:00:00 2001 From: Adriaan de Groot Date: Tue, 19 Jun 2018 08:46:50 -0400 Subject: [PATCH 3/7] [localecfg] Move all path-setting to one spot - Make the way the paths are constructed consistent - Name the paths more consistently --- src/modules/localecfg/main.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/src/modules/localecfg/main.py b/src/modules/localecfg/main.py index 55c704fae9..a192fa9024 100644 --- a/src/modules/localecfg/main.py +++ b/src/modules/localecfg/main.py @@ -48,23 +48,27 @@ def run(): } install_path = libcalamares.globalstorage.value("rootMountPoint") + target_locale_gen = "{!s}/etc/locale.gen".format(install_path) + target_locale_gen_bak = target_locale_gen + ".bak" + target_locale_conf_path = "{!s}/etc/locale.conf".format(install_path) + target_etc_default_path = "{!s}/etc/default".format(install_path) # restore backup if available if os.path.exists('/etc/locale.gen.bak'): - shutil.copy2("{!s}/etc/locale.gen.bak".format(install_path), - "{!s}/etc/locale.gen".format(install_path)) + shutil.copy2(target_locale_gen_bak, + target_locale_gen) # run locale-gen if detected if os.path.exists('/etc/locale.gen'): text = [] - with open("{!s}/etc/locale.gen".format(install_path), "r") as gen: + with open(target_locale_gen, "r") as gen: text = gen.readlines() # we want unique values, so locale_values should have 1 or 2 items locale_values = set(locale_conf.values()) - with open("{!s}/etc/locale.gen".format(install_path), "w") as gen: + with open(target_locale_gen, "w") as gen: for line in text: # always enable en_US if line.startswith("#" + en_us_locale): @@ -82,15 +86,13 @@ def run(): print('locale.gen done') # write /etc/locale.conf - locale_conf_path = os.path.join(install_path, "etc/locale.conf") - with open(locale_conf_path, "w") as lcf: + with open(target_locale_conf_path, "w") as lcf: for k, v in locale_conf.items(): lcf.write("{!s}={!s}\n".format(k, v)) # write /etc/default/locale if /etc/default exists and is a dir - etc_default_path = os.path.join(install_path, "etc/default") - if os.path.isdir(etc_default_path): - with open(os.path.join(etc_default_path, "locale"), "w") as edl: + if os.path.isdir(target_etc_default_path): + with open(os.path.join(target_etc_default_path, "locale"), "w") as edl: for k, v in locale_conf.items(): edl.write("{!s}={!s}\n".format(k, v)) From efc977f7b437d11011de640659b42e4bb78e9708 Mon Sep 17 00:00:00 2001 From: Adriaan de Groot Date: Tue, 19 Jun 2018 08:56:37 -0400 Subject: [PATCH 4/7] [localecfg] Fix mismatch between filenames Testing for existence of a file in the live system, and then copying it in the target system, is not a recipe for success. - Fix the restore-from-backup part. - Document that your live and target system must both have /etc/locale.gen if you want this to work at all. --- src/modules/localecfg/main.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/modules/localecfg/main.py b/src/modules/localecfg/main.py index a192fa9024..4b12b9311c 100644 --- a/src/modules/localecfg/main.py +++ b/src/modules/localecfg/main.py @@ -54,11 +54,12 @@ def run(): target_etc_default_path = "{!s}/etc/default".format(install_path) # restore backup if available - if os.path.exists('/etc/locale.gen.bak'): - shutil.copy2(target_locale_gen_bak, - target_locale_gen) + if os.path.exists(target_locale_gen_bak): + shutil.copy2(target_locale_gen_bak, target_locale_gen) - # run locale-gen if detected + # run locale-gen if detected; this *will* cause an exception + # if the live system has locale.gen, but the target does not: + # in that case, fix your installation filesystem. if os.path.exists('/etc/locale.gen'): text = [] From 413ee81eade994a99ab3894c4bafaa4b146df931 Mon Sep 17 00:00:00 2001 From: Adriaan de Groot Date: Wed, 20 Jun 2018 07:13:36 -0400 Subject: [PATCH 5/7] [localecfg] Simplify handling of en_US - By adding en_US to the set of locales-to-enable, we can drop the special-case code for it. --- src/modules/localecfg/main.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/modules/localecfg/main.py b/src/modules/localecfg/main.py index 4b12b9311c..413026c06e 100644 --- a/src/modules/localecfg/main.py +++ b/src/modules/localecfg/main.py @@ -68,14 +68,10 @@ def run(): # we want unique values, so locale_values should have 1 or 2 items locale_values = set(locale_conf.values()) + locale_values.add(en_us_locale) # Always enable en_US as well with open(target_locale_gen, "w") as gen: for line in text: - # always enable en_US - if line.startswith("#" + en_us_locale): - # uncomment line - line = line[1:].lstrip() - for locale_value in locale_values: if line.startswith("#" + locale_value): # uncomment line From 85516535754bbbafb6d53bebf3e73c5e65e4e160 Mon Sep 17 00:00:00 2001 From: Adriaan de Groot Date: Wed, 20 Jun 2018 08:35:18 -0400 Subject: [PATCH 6/7] [localecfg] Re-write the locale.gen file better - Improved debug-logging - Fix the actual problem of listing locales more than once, by listing them all, uniqified, at the end, with an explanitory comment in the generated file. - Be more accepting of what constitutes a locale-line; this allows spaces before and after the `#` comment sign, but because we're uniquifying, this doesn't cause duplicates. - Because we write the enabled locales at the end, the full file comment-header is retained un-mangled (instead of accidentally enabling a locale mentioned as an example there). --- src/modules/localecfg/main.py | 100 +++++++++++++++++++++++++++------- 1 file changed, 79 insertions(+), 21 deletions(-) diff --git a/src/modules/localecfg/main.py b/src/modules/localecfg/main.py index 413026c06e..bf41e6317b 100644 --- a/src/modules/localecfg/main.py +++ b/src/modules/localecfg/main.py @@ -23,14 +23,85 @@ # along with Calamares. If not, see . import os +import re import shutil -import libcalamares +RE_IS_COMMENT = re.compile("^ *#") +def is_comment(line): + """ + Does the @p line look like a comment? Whitespace, followed by a # + is a comment-only line. + """ + return bool(RE_IS_COMMENT.match(line)) + +RE_REST_OF_LINE = re.compile("\\s.*$") +def extract_locale(line): + """ + Extracts a locale from the @p line, and returns a pair of + (extracted-locale, uncommented line). The locale is the + first word of the line after uncommenting (in the human- + readable text explanation at the top of most /etc/locale.gen + files, the locales may be bogus -- either "" or e.g. "Configuration") + """ + # Remove leading spaces and comment signs + line = RE_IS_COMMENT.sub("", line) + uncommented = line.strip() + # Drop all but first field + locale = RE_REST_OF_LINE.sub("", uncommented) + return locale, uncommented + + +def rewrite_locale_gen(srcfilename, destfilename, locale_conf): + """ + Copies a locale.gen file from @p srcfilename to @p destfilename + (this may be the same name), enabling those locales that can + be found in the map @p locale_conf. Also always enables en_US.UTF-8. + """ + en_us_locale = 'en_US.UTF-8' + + # Get entire source-file contents + text = [] + with open(srcfilename, "r") as gen: + text = gen.readlines() + + # we want unique values, so locale_values should have 1 or 2 items + locale_values = set(locale_conf.values()) + locale_values.add(en_us_locale) # Always enable en_US as well + + enabled_locales = {} + seen_locales = set() + + # Write source out again, enabling some + with open(destfilename, "w") as gen: + for line in text: + c = is_comment(line) + locale, uncommented = extract_locale(line) + + # Non-comment lines are preserved, and comment lines + # may be enabled if they match a desired locale + if not c: + seen_locales.add(locale) + else: + for locale_value in locale_values: + if locale.startswith(locale_value): + enabled_locales[locale] = uncommented + gen.write(line) + + gen.write("\n###\n#\n# Locales enabled by Calamares\n") + for locale, line in enabled_locales.items(): + if locale not in seen_locales: + gen.write(line + "\n") + seen_locales.add(locale) + + for locale in locale_values: + if locale not in seen_locales: + gen.write("# Missing: %s\n" % locale) def run(): """ Create locale """ - en_us_locale = 'en_US.UTF-8' + import libcalamares + locale_conf = libcalamares.globalstorage.value("localeConf") if not locale_conf: @@ -56,41 +127,28 @@ def run(): # restore backup if available if os.path.exists(target_locale_gen_bak): shutil.copy2(target_locale_gen_bak, target_locale_gen) + libcalamares.utils.debug("Restored backup {!s} -> {!s}" + .format(target_locale_gen_bak).format(target_locale_gen)) # run locale-gen if detected; this *will* cause an exception # if the live system has locale.gen, but the target does not: # in that case, fix your installation filesystem. if os.path.exists('/etc/locale.gen'): - text = [] - - with open(target_locale_gen, "r") as gen: - text = gen.readlines() - - # we want unique values, so locale_values should have 1 or 2 items - locale_values = set(locale_conf.values()) - locale_values.add(en_us_locale) # Always enable en_US as well - - with open(target_locale_gen, "w") as gen: - for line in text: - for locale_value in locale_values: - if line.startswith("#" + locale_value): - # uncomment line - line = line[1:].lstrip() - - gen.write(line) - + rewrite_locale_gen(target_locale_gen, target_locale_gen, locale_conf) libcalamares.utils.target_env_call(['locale-gen']) - print('locale.gen done') + libcalamares.utils.debug('{!s} done'.format(target_locale_gen)) # write /etc/locale.conf with open(target_locale_conf_path, "w") as lcf: for k, v in locale_conf.items(): lcf.write("{!s}={!s}\n".format(k, v)) + libcalamares.utils.debug('{!s} done'.format(target_locale_conf_path)) # write /etc/default/locale if /etc/default exists and is a dir if os.path.isdir(target_etc_default_path): with open(os.path.join(target_etc_default_path, "locale"), "w") as edl: for k, v in locale_conf.items(): edl.write("{!s}={!s}\n".format(k, v)) + libcalamares.utils.debug('{!s} done'.format(target_etc_default_path)) return None From 25f249180b40f95958601f5d2431a365bd6ce2a8 Mon Sep 17 00:00:00 2001 From: Adriaan de Groot Date: Wed, 20 Jun 2018 09:11:23 -0400 Subject: [PATCH 7/7] [localecfg] Be slightly more conservative interpreting comments - A valid line (as explained in the comments at the top of the locale.gen file) is (two fields), so lines with more than two fields can't be valid locale- listing lines. For them, pretend they name locale "", which won't be matched. --- src/modules/localecfg/main.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/modules/localecfg/main.py b/src/modules/localecfg/main.py index bf41e6317b..62a00b738d 100644 --- a/src/modules/localecfg/main.py +++ b/src/modules/localecfg/main.py @@ -34,6 +34,7 @@ def is_comment(line): """ return bool(RE_IS_COMMENT.match(line)) +RE_TRAILING_COMMENT = re.compile("#.*$") RE_REST_OF_LINE = re.compile("\\s.*$") def extract_locale(line): """ @@ -46,9 +47,14 @@ def extract_locale(line): # Remove leading spaces and comment signs line = RE_IS_COMMENT.sub("", line) uncommented = line.strip() - # Drop all but first field - locale = RE_REST_OF_LINE.sub("", uncommented) - return locale, uncommented + fields = RE_TRAILING_COMMENT.sub("", uncommented).strip().split() + if len(fields) != 2: + # Not exactly two fields, can't be a proper locale line + return "", uncommented + else: + # Drop all but first field + locale = RE_REST_OF_LINE.sub("", uncommented) + return locale, uncommented def rewrite_locale_gen(srcfilename, destfilename, locale_conf):