Skip to content

Commit

Permalink
Fix encoding issues with config files + PY3 support for --configure
Browse files Browse the repository at this point in the history 
- Fixes --configure with PY3 as "raw_input" doesn't exist anymore in PY3
- Fixes for encoding/utf8 issues with config files
- Exit with the proper error if an error is detected with the config
file.
  • Loading branch information
@fviard
fviard committed Aug 5, 2017
1 parent ae957a4 commit 97482c2
Show file tree
Hide file tree
Showing 2 changed files with 60 additions and 50 deletions.
68 changes: 32 additions & 36 deletions S3/Config.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
from logging import debug, warning, error
import re
import os
import io
import sys
import json
from . import Progress
Expand Down Expand Up @@ -284,15 +285,15 @@ def read_config_file(self, configfile):
self._parsed_files.append(configfile)

def dump_config(self, stream):
ConfigDumper(stream).dump("default", self)
ConfigDumper(stream).dump(u"default", self)

def update_option(self, option, value):
if value is None:
return

#### Handle environment reference
if str(value).startswith("$"):
return self.update_option(option, os.getenv(str(value)[1:]))
if value.startswith("$"):
return self.update_option(option, os.getenv(value[1:]))

#### Special treatment of some options
## verbosity must be known to "logging" module
Expand All @@ -309,8 +310,7 @@ def update_option(self, option, value):
except AttributeError:
value = logging._nameToLevel[value]
except KeyError:
error("Config: verbosity level '%s' is not valid" % value)
return
raise ValueError("Config: verbosity level '%s' is not valid" % value)

elif option == "limitrate":
#convert kb,mb to bytes
Expand All @@ -323,8 +323,7 @@ def update_option(self, option, value):
try:
value = shift and int(value[:-1]) << shift or int(value)
except:
error("Config: value of option %s must have suffix m, k, or nothing, not '%s'" % (option, value))
return
raise ValueError("Config: value of option %s must have suffix m, k, or nothing, not '%s'" % (option, value))

## allow yes/no, true/false, on/off and 1/0 for boolean options
elif type(getattr(Config, option)) is type(True): # bool
Expand All @@ -333,15 +332,13 @@ def update_option(self, option, value):
elif str(value).lower() in ("false", "no", "off", "0"):
value = False
else:
error("Config: value of option '%s' must be Yes or No, not '%s'" % (option, value))
return
raise ValueError("Config: value of option '%s' must be Yes or No, not '%s'" % (option, value))

elif type(getattr(Config, option)) is type(42): # int
try:
value = int(value)
except ValueError:
error("Config: value of option '%s' must be an integer, not '%s'" % (option, value))
return
raise ValueError("Config: value of option '%s' must be an integer, not '%s'" % (option, value))

elif option in ["host_base", "host_bucket", "cloudfront_host"]:
if value.startswith("http://"):
Expand All @@ -362,33 +359,33 @@ def parse_file(self, file, sections = []):
if type(sections) != type([]):
sections = [sections]
in_our_section = True
f = open(file, "r")
r_comment = re.compile("^\s*#.*")
r_empty = re.compile("^\s*$")
r_section = re.compile("^\[([^\]]+)\]")
r_data = re.compile("^\s*(?P<key>\w+)\s*=\s*(?P<value>.*)")
r_quotes = re.compile("^\"(.*)\"\s*$")
for line in f:
if r_comment.match(line) or r_empty.match(line):
continue
is_section = r_section.match(line)
if is_section:
section = is_section.groups()[0]
in_our_section = (section in sections) or (len(sections) == 0)
continue
is_data = r_data.match(line)
if is_data and in_our_section:
data = is_data.groupdict()
if r_quotes.match(data["value"]):
data["value"] = data["value"][1:-1]
self.__setitem__(data["key"], data["value"])
if data["key"] in ("access_key", "secret_key", "gpg_passphrase"):
print_value = ("%s...%d_chars...%s") % (data["value"][:2], len(data["value"]) - 3, data["value"][-1:])
else:
print_value = data["value"]
debug("ConfigParser: %s->%s" % (data["key"], print_value))
continue
warning("Ignoring invalid line in '%s': %s" % (file, line))
with io.open(file, "r", encoding=self.get('encoding', 'UTF-8')) as fp:
for line in fp:
if r_comment.match(line) or r_empty.match(line):
continue
is_section = r_section.match(line)
if is_section:
section = is_section.groups()[0]
in_our_section = (section in sections) or (len(sections) == 0)
continue
is_data = r_data.match(line)
if is_data and in_our_section:
data = is_data.groupdict()
if r_quotes.match(data["value"]):
data["value"] = data["value"][1:-1]
self.__setitem__(data["key"], data["value"])
if data["key"] in ("access_key", "secret_key", "gpg_passphrase"):
print_value = ("%s...%d_chars...%s") % (data["value"][:2], len(data["value"]) - 3, data["value"][-1:])
else:
print_value = data["value"]
debug("ConfigParser: %s->%s" % (data["key"], print_value))
continue
warning("Ignoring invalid line in '%s': %s" % (file, line))

def __getitem__(self, name):
return self.cfg[name]
Expand All @@ -406,7 +403,7 @@ def __init__(self, stream):
self.stream = stream

def dump(self, section, config):
self.stream.write("[%s]\n" % section)
self.stream.write(u"[%s]\n" % section)
for option in config.option_list():
value = getattr(config, option)
if option == "verbosity":
Expand All @@ -420,7 +417,6 @@ def dump(self, section, config):
value = logging._levelToName[value]
except KeyError:
pass

self.stream.write("%s = %s\n" % (option, value))
self.stream.write(u"%s = %s\n" % (option, value))

# vim:et:ts=4:sts=4:ai
42 changes: 28 additions & 14 deletions s3cmd
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2235,7 +2235,15 @@ def run_configure(config_file, args):
setattr(cfg, "proxy_port", re_match.groups()[2])

try:
while 1:
# Support for python3
# raw_input only exists in py2 and was renamed to input in py3
global input
input = raw_input
except NameError:
pass

try:
while True:
output(u"\nEnter new values or accept defaults in brackets with Enter.")
output(u"Refer to user manual for detailed description of all options.")
for option in options:
Expand All @@ -2260,7 +2268,7 @@ def run_configure(config_file, args):
if len(option) >= 3:
output(u"\n%s" % option[2])

val = raw_input(prompt + ": ")
val = unicodise_s(input(prompt + ": "))
if val != "":
if type(getattr(cfg, option[0])) is bool:
# Turn 'Yes' into True, everything else into False
Expand All @@ -2269,7 +2277,7 @@ def run_configure(config_file, args):
output(u"\nNew settings:")
for option in options:
output(u" %s: %s" % (option[1], getattr(cfg, option[0])))
val = raw_input("\nTest access with supplied credentials? [Y/n] ")
val = input("\nTest access with supplied credentials? [Y/n] ")
if val.lower().startswith("y") or val == "":
try:
# Default, we try to list 'all' buckets which requires
Expand Down Expand Up @@ -2299,9 +2307,8 @@ def run_configure(config_file, args):
if not os.path.isfile(deunicodise(getattr(cfg, "gpg_command"))):
raise Exception("GPG program not found")
filename = Utils.mktmpfile()
f = open(deunicodise(filename), "w")
f.write(os.sys.copyright)
f.close()
with open(deunicodise(filename), "w") as fp:
fp.write(os.sys.copyright)
ret_enc = gpg_encrypt(filename)
ret_dec = gpg_decrypt(ret_enc[1], ret_enc[2], False)
hash = [
Expand All @@ -2321,20 +2328,20 @@ def run_configure(config_file, args):
error(u"Test failed: %s" % (e))
if e.code == "AccessDenied":
error(u"Are you sure your keys have s3:ListAllMyBuckets permissions?")
val = raw_input("\nRetry configuration? [Y/n] ")
val = input("\nRetry configuration? [Y/n] ")
if val.lower().startswith("y") or val == "":
continue
except Exception as e:
error(u"Test failed: %s" % (e))
val = raw_input("\nRetry configuration? [Y/n] ")
val = input("\nRetry configuration? [Y/n] ")
if val.lower().startswith("y") or val == "":
continue


val = raw_input("\nSave settings? [y/N] ")
val = input("\nSave settings? [y/N] ")
if val.lower().startswith("y"):
break
val = raw_input("Retry configuration? [Y/n] ")
val = input("Retry configuration? [Y/n] ")
if val.lower().startswith("n"):
raise EOFError()

Expand All @@ -2345,10 +2352,11 @@ def run_configure(config_file, args):
except OSError as e:
if e.errno != errno.ENOENT:
raise
f = open(config_file, "w")
os.umask(old_mask)
cfg.dump_config(f)
f.close()
try:
with io.open(deunicodise(config_file), "w", encoding=cfg.encoding) as fp:
cfg.dump_config(fp)
finally:
os.umask(old_mask)
output(u"Configuration saved to '%s'" % config_file)

except (EOFError, KeyboardInterrupt):
Expand Down Expand Up @@ -2724,6 +2732,8 @@ def main():

try:
cfg = Config(options.config, options.access_key, options.secret_key, options.access_token)
except ValueError as exc:
raise ParameterError(unicode(exc))
except IOError as e:
if options.run_configure:
cfg = Config()
Expand Down Expand Up @@ -2846,6 +2856,10 @@ def main():
# of stdout/stderr
sys.stdout = codecs.getwriter(cfg.encoding)(sys.stdout.buffer, "replace")
sys.stderr = codecs.getwriter(cfg.encoding)(sys.stderr.buffer, "replace")
# getwriter with create an "IObuffer" that have not the encoding attribute
# better to add it to not break some functions like "input".
sys.stdout.encoding = cfg.encoding
sys.stderr.encoding = cfg.encoding
except AttributeError:
sys.stdout = codecs.getwriter(cfg.encoding)(sys.stdout, "replace")
sys.stderr = codecs.getwriter(cfg.encoding)(sys.stderr, "replace")
Expand Down

0 comments on commit 97482c2

Please sign in to comment.