From ebbaa22bc85f0cf424d17c787bc898894dbcc0a4 Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Tue, 21 Nov 2017 12:00:21 -0500 Subject: [PATCH 01/75] bpo-32107 - Fix test_uuid The original check was erroneous in two ways. First, the documentation said that "47 bit will never be set in IEEE 802 addresses obtained from network cards". I think this is referring to the universally vs. locally administered MAC addresses. Network cards from hardware manufacturers will always be universally administered in order to guarantee global uniqueness. But from https://en.wikipedia.org/wiki/MAC_address it says that this flag is the "distinguished by setting the second-least-significant bit of the first octet of the address", where a 0 means universally administered and a 1 means it's locally administered. This works out to the 42nd bit when counting the LSB as bit 1, or 1<<41. The second bug is that the original bitmask value isn't right for either description: % ./python.exe -c "from math import log2; print(log2(0x010000000000))" 40.0 This causes the test to fail on valid MAC addresses. Fix this by improving the comment, with references, and using the correct bitmask. --- Lib/test/test_uuid.py | 13 ++++++++++--- Lib/uuid.py | 3 +-- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index 083c2aa8aab54d..5adba57e07d9f2 100644 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -519,9 +519,16 @@ def check_node(self, node, requires=None, network=False): if support.verbose >= 2: print(hex, end=' ') if network: - # 47 bit will never be set in IEEE 802 addresses obtained - # from network cards. - self.assertFalse(node & 0x010000000000, hex) + # The second least significant bit of the first octet signifies + # whether the MAC (IEEE 802, or EUI-48) address is universally (0) + # or locally (1) administered. Network cards from hardware + # manufacturers will always be universally administered to + # guarantee global uniqueness of the MAC address. This bit works + # out to be the 42nd bit counting from 1 being the least + # significant, or 1<<41. For a good, simple explanation, see the + # section on Universal vs. local in this page: + # https://en.wikipedia.org/wiki/MAC_address + self.assertFalse(node & (1 << 41), hex) self.assertTrue(0 < node < (1 << 48), "%s is not an RFC 4122 node ID" % hex) diff --git a/Lib/uuid.py b/Lib/uuid.py index 020c6e73c863d4..fee809dd1abfab 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -404,8 +404,7 @@ def _arp_getnode(): # This works on Linux, FreeBSD and NetBSD mac = _find_mac('arp', '-an', [os.fsencode('(%s)' % ip_addr)], lambda i: i+2) - if mac: - return mac + return mac def _lanscan_getnode(): """Get the hardware address on Unix by running lanscan.""" From 941b47db8e2cb1587951730f10ecdbfdc48585b5 Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Tue, 21 Nov 2017 14:10:40 -0500 Subject: [PATCH 02/75] Return None instead of 0 --- Lib/uuid.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/uuid.py b/Lib/uuid.py index fee809dd1abfab..27335f89192909 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -404,7 +404,8 @@ def _arp_getnode(): # This works on Linux, FreeBSD and NetBSD mac = _find_mac('arp', '-an', [os.fsencode('(%s)' % ip_addr)], lambda i: i+2) - return mac + # Return None instead of 0. + return mac if mac else None def _lanscan_getnode(): """Get the hardware address on Unix by running lanscan.""" From 803ddd8ce22f0de3ab42fb98a225a704c000ef06 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 21 Nov 2017 15:34:02 -0800 Subject: [PATCH 03/75] bpo-31324: Optimize support._match_test() (#4421) * Rename support._match_test() to support.match_test(): make it public * Remove support.match_tests global variable. It is replaced with a new support.set_match_tests() function, so match_test() doesn't have to check each time if patterns were modified. * Rewrite match_test(): use different code paths depending on the kind of patterns for best performances. Co-Authored-By: Serhiy Storchaka --- Lib/test/libregrtest/main.py | 4 +- Lib/test/libregrtest/runtest.py | 2 +- Lib/test/support/__init__.py | 67 +++++++++++++++++++++++++++------ Lib/test/test_support.py | 53 ++++++++++++++++++++++++++ 4 files changed, 111 insertions(+), 15 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 9871a28dbf2d2b..ce01c8ce586d65 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -257,12 +257,12 @@ def _list_cases(self, suite): if isinstance(test, unittest.TestSuite): self._list_cases(test) elif isinstance(test, unittest.TestCase): - if support._match_test(test): + if support.match_test(test): print(test.id()) def list_cases(self): support.verbose = False - support.match_tests = self.ns.match_tests + support.set_match_tests(self.ns.match_tests) for test in self.selected: abstest = get_abs_module(self.ns, test) diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index dbd463435c781b..12bf422c902dc1 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -102,7 +102,7 @@ def runtest(ns, test): if use_timeout: faulthandler.dump_traceback_later(ns.timeout, exit=True) try: - support.match_tests = ns.match_tests + support.set_match_tests(ns.match_tests) # reset the environment_altered flag to detect if a test altered # the environment support.environment_altered = False diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 527cf7fbf95328..71d9c2c8795905 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -278,7 +278,6 @@ def get_attribute(obj, name): # small sizes, to make sure they work.) real_max_memuse = 0 failfast = False -match_tests = None # _original_stdout is meant to hold stdout at the time regrtest began. # This may be "the real" stdout, or IDLE's emulation of stdout, or whatever. @@ -1900,21 +1899,65 @@ def _run_suite(suite): raise TestFailed(err) -def _match_test(test): - global match_tests +# By default, don't filter tests +_match_test_func = None +_match_test_patterns = None - if match_tests is None: + +def match_test(test): + # Function used by support.run_unittest() and regrtest --list-cases + if _match_test_func is None: return True - test_id = test.id() + else: + return _match_test_func(test.id()) - for match_test in match_tests: - if fnmatch.fnmatchcase(test_id, match_test): - return True - for name in test_id.split("."): - if fnmatch.fnmatchcase(name, match_test): +def _is_full_match_test(pattern): + # If a pattern contains at least one dot, it's considered + # as a full test identifier. + # Example: 'test.test_os.FileTests.test_access'. + # + # Reject patterns which contain fnmatch patterns: '*', '?', '[...]' + # or '[!...]'. For example, reject 'test_access*'. + return ('.' in pattern) and (not re.search(r'[?*\[\]]', pattern)) + + +def set_match_tests(patterns): + global _match_test_func, _match_test_patterns + + if patterns == _match_test_patterns: + # No change: no need to recompile patterns. + return + + if not patterns: + func = None + elif all(map(_is_full_match_test, patterns)): + # Simple case: all patterns are full test identifier. + # The test.bisect utility only uses such full test identifiers. + func = set(patterns).__contains__ + else: + regex = '|'.join(map(fnmatch.translate, patterns)) + # The search *is* case sensitive on purpose: + # don't use flags=re.IGNORECASE + regex_match = re.compile(regex).match + + def match_test_regex(test_id): + if regex_match(test_id): + # The regex matchs the whole identifier like + # 'test.test_os.FileTests.test_access' return True - return False + else: + # Try to match parts of the test identifier. + # For example, split 'test.test_os.FileTests.test_access' + # into: 'test', 'test_os', 'FileTests' and 'test_access'. + return any(map(regex_match, test_id.split("."))) + + func = match_test_regex + + # Create a copy since patterns can be mutable and so modified later + _match_test_patterns = tuple(patterns) + _match_test_func = func + def run_unittest(*classes): @@ -1931,7 +1974,7 @@ def run_unittest(*classes): suite.addTest(cls) else: suite.addTest(unittest.makeSuite(cls)) - _filter_suite(suite, _match_test) + _filter_suite(suite, match_test) _run_suite(suite) #======================================================================= diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index 4a577efbeb9ccb..4756defa5f79b7 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -483,6 +483,59 @@ def test_optim_args_from_interpreter_flags(self): with self.subTest(opts=opts): self.check_options(opts, 'optim_args_from_interpreter_flags') + def test_match_test(self): + class Test: + def __init__(self, test_id): + self.test_id = test_id + + def id(self): + return self.test_id + + test_access = Test('test.test_os.FileTests.test_access') + test_chdir = Test('test.test_os.Win32ErrorTests.test_chdir') + + with support.swap_attr(support, '_match_test_func', None): + # match all + support.set_match_tests([]) + self.assertTrue(support.match_test(test_access)) + self.assertTrue(support.match_test(test_chdir)) + + # match the full test identifier + support.set_match_tests([test_access.id()]) + self.assertTrue(support.match_test(test_access)) + self.assertFalse(support.match_test(test_chdir)) + + # match the module name + support.set_match_tests(['test_os']) + self.assertTrue(support.match_test(test_access)) + self.assertTrue(support.match_test(test_chdir)) + + # Test '*' pattern + support.set_match_tests(['test_*']) + self.assertTrue(support.match_test(test_access)) + self.assertTrue(support.match_test(test_chdir)) + + # Test case sensitivity + support.set_match_tests(['filetests']) + self.assertFalse(support.match_test(test_access)) + support.set_match_tests(['FileTests']) + self.assertTrue(support.match_test(test_access)) + + # Test pattern containing '.' and a '*' metacharacter + support.set_match_tests(['*test_os.*.test_*']) + self.assertTrue(support.match_test(test_access)) + self.assertTrue(support.match_test(test_chdir)) + + # Multiple patterns + support.set_match_tests([test_access.id(), test_chdir.id()]) + self.assertTrue(support.match_test(test_access)) + self.assertTrue(support.match_test(test_chdir)) + + support.set_match_tests(['test_access', 'DONTMATCH']) + self.assertTrue(support.match_test(test_access)) + self.assertFalse(support.match_test(test_chdir)) + + # XXX -follows a list of untested API # make_legacy_pyc # is_resource_enabled From e529214489ed802468fab5ea41642bc1a38c3ad9 Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Tue, 21 Nov 2017 23:08:06 -0500 Subject: [PATCH 04/75] Never return a locally administered MAC address Also, clean up some return sites. --- Lib/test/test_uuid.py | 9 --------- Lib/uuid.py | 45 ++++++++++++++++++++++++++++++++++--------- 2 files changed, 36 insertions(+), 18 deletions(-) diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index 5adba57e07d9f2..19564f2d548d28 100644 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -519,15 +519,6 @@ def check_node(self, node, requires=None, network=False): if support.verbose >= 2: print(hex, end=' ') if network: - # The second least significant bit of the first octet signifies - # whether the MAC (IEEE 802, or EUI-48) address is universally (0) - # or locally (1) administered. Network cards from hardware - # manufacturers will always be universally administered to - # guarantee global uniqueness of the MAC address. This bit works - # out to be the 42nd bit counting from 1 being the least - # significant, or 1<<41. For a good, simple explanation, see the - # section on Universal vs. local in this page: - # https://en.wikipedia.org/wiki/MAC_address self.assertFalse(node & (1 << 41), hex) self.assertTrue(0 < node < (1 << 48), "%s is not an RFC 4122 node ID" % hex) diff --git a/Lib/uuid.py b/Lib/uuid.py index 27335f89192909..0535433f2bffda 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -342,11 +342,26 @@ def _popen(command, *args): env=env) return proc +# For MAC (a.k.a. IEEE 802, or EUI-48) addresses, the second least significant +# bit of the first octet signifies whether the MAC address is universally (0) +# or locally (1) administered. Network cards from hardware manufacturers will +# always be universally administered to guarantee global uniqueness of the MAC +# address, but any particular machine may have other interfaces which are +# locally administered. An example of the latter is the bridge interface to +# the Touch Bar on MacBook Pros. +# +# This bit works out to be the 42nd bit counting from 1 being the least +# significant, or 1<<41. We'll skip over any locally administered MAC +# addresses, as it makes no sense to use those in UUID calculation. +# +# For a good, simple explanation, see the section on Universal vs. local in +# this page: https://en.wikipedia.org/wiki/MAC_address + def _find_mac(command, args, hw_identifiers, get_index): try: proc = _popen(command, *args.split()) if not proc: - return + return None with proc: for line in proc.stdout: words = line.lower().rstrip().split() @@ -355,7 +370,7 @@ def _find_mac(command, args, hw_identifiers, get_index): try: word = words[get_index(i)] mac = int(word.replace(b':', b''), 16) - if mac: + if ~(mac & (1<<41)): return mac except (ValueError, IndexError): # Virtual interfaces, such as those provided by @@ -366,6 +381,7 @@ def _find_mac(command, args, hw_identifiers, get_index): pass except OSError: pass + return None def _ifconfig_getnode(): """Get the hardware address on Unix by running ifconfig.""" @@ -375,6 +391,7 @@ def _ifconfig_getnode(): mac = _find_mac('ifconfig', args, keywords, lambda i: i+1) if mac: return mac + return None def _ip_getnode(): """Get the hardware address on Unix by running ip.""" @@ -382,6 +399,7 @@ def _ip_getnode(): mac = _find_mac('ip', 'link list', [b'link/ether'], lambda i: i+1) if mac: return mac + return None def _arp_getnode(): """Get the hardware address on Unix by running arp.""" @@ -405,7 +423,9 @@ def _arp_getnode(): mac = _find_mac('arp', '-an', [os.fsencode('(%s)' % ip_addr)], lambda i: i+2) # Return None instead of 0. - return mac if mac else None + if mac: + return mac + return None def _lanscan_getnode(): """Get the hardware address on Unix by running lanscan.""" @@ -418,25 +438,26 @@ def _netstat_getnode(): try: proc = _popen('netstat', '-ia') if not proc: - return + return None with proc: words = proc.stdout.readline().rstrip().split() try: i = words.index(b'Address') except ValueError: - return + return None for line in proc.stdout: try: words = line.rstrip().split() word = words[i] if len(word) == 17 and word.count(b':') == 5: mac = int(word.replace(b':', b''), 16) - if mac: + if ~(mac & (1<<41)): return mac except (ValueError, IndexError): pass except OSError: pass + return None def _ipconfig_getnode(): """Get the hardware address on Windows by running ipconfig.exe.""" @@ -458,7 +479,10 @@ def _ipconfig_getnode(): for line in pipe: value = line.split(':')[-1].strip().lower() if re.match('([0-9a-f][0-9a-f]-){5}[0-9a-f][0-9a-f]', value): - return int(value.replace('-', ''), 16) + mac = int(value.replace('-', ''), 16) + if ~(mac & (1<<41)): + return mac + return None def _netbios_getnode(): """Get the hardware address on Windows using NetBIOS calls. @@ -469,7 +493,7 @@ def _netbios_getnode(): ncb.Buffer = adapters = netbios.LANA_ENUM() adapters._pack() if win32wnet.Netbios(ncb) != 0: - return + return None adapters._unpack() for i in range(adapters.length): ncb.Reset() @@ -488,7 +512,10 @@ def _netbios_getnode(): bytes = status.adapter_address[:6] if len(bytes) != 6: continue - return int.from_bytes(bytes, 'big') + mac = int.from_bytes(bytes, 'big') + if ~(mac & (1<<41)): + return mac + return None _generate_time_safe = _UuidCreate = None From bb11c3c967afaf263e00844d4ab461b7fafd6d36 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 22 Nov 2017 20:58:59 +0100 Subject: [PATCH 05/75] bpo-31324: Fix test.support.set_match_tests(None) (#4505) --- Lib/test/support/__init__.py | 2 ++ Lib/test/test_support.py | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 71d9c2c8795905..b7cbdc6ab30542 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -1931,6 +1931,8 @@ def set_match_tests(patterns): if not patterns: func = None + # set_match_tests(None) behaves as set_match_tests(()) + patterns = () elif all(map(_is_full_match_test, patterns)): # Simple case: all patterns are full test identifier. # The test.bisect utility only uses such full test identifiers. diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index 4756defa5f79b7..e06f7b8e9952b8 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -500,6 +500,11 @@ def id(self): self.assertTrue(support.match_test(test_access)) self.assertTrue(support.match_test(test_chdir)) + # match all using None + support.set_match_tests(None) + self.assertTrue(support.match_test(test_access)) + self.assertTrue(support.match_test(test_chdir)) + # match the full test identifier support.set_match_tests([test_access.id()]) self.assertTrue(support.match_test(test_access)) From 82656276caf4cb889193572d2d14dbc5f3d2bdff Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 22 Nov 2017 23:51:42 +0100 Subject: [PATCH 06/75] bpo-27535: Optimize warnings.warn() (#4508) * Optimize warnings.filterwarnings(). Replace re.compile('') with None to avoid the cost of calling a regex.match() method, whereas it always matchs. * Optimize get_warnings_attr(): replace PyObject_GetAttrString() with _PyObject_GetAttrId(). Cleanup also create_filter(): * Use _Py_IDENTIFIER() to allow to cleanup strings at Python finalization * Replace Py_FatalError() with a regular exceptions --- Lib/warnings.py | 17 +++++++++-- Python/_warnings.c | 75 ++++++++++++++++++++-------------------------- 2 files changed, 47 insertions(+), 45 deletions(-) diff --git a/Lib/warnings.py b/Lib/warnings.py index b2605f84aec0f5..5b62569c977350 100644 --- a/Lib/warnings.py +++ b/Lib/warnings.py @@ -128,7 +128,6 @@ def filterwarnings(action, message="", category=Warning, module="", lineno=0, 'lineno' -- an integer line number, 0 matches all warnings 'append' -- if true, append to the list of filters """ - import re assert action in ("error", "ignore", "always", "default", "module", "once"), "invalid action: %r" % (action,) assert isinstance(message, str), "message must be a string" @@ -137,8 +136,20 @@ def filterwarnings(action, message="", category=Warning, module="", lineno=0, assert isinstance(module, str), "module must be a string" assert isinstance(lineno, int) and lineno >= 0, \ "lineno must be an int >= 0" - _add_filter(action, re.compile(message, re.I), category, - re.compile(module), lineno, append=append) + + if message or module: + import re + + if message: + message = re.compile(message, re.I) + else: + message = None + if module: + module = re.compile(module) + else: + module = None + + _add_filter(action, message, category, module, lineno, append=append) def simplefilter(action, category=Warning, lineno=0, append=False): """Insert a simple entry into the list of warnings filters (at the front). diff --git a/Python/_warnings.c b/Python/_warnings.c index f2110edc52d196..d865f0ad2c089a 100644 --- a/Python/_warnings.c +++ b/Python/_warnings.c @@ -35,15 +35,15 @@ check_matched(PyObject *obj, PyObject *arg) A NULL return value can mean false or an error. */ static PyObject * -get_warnings_attr(const char *attr, int try_import) +get_warnings_attr(_Py_Identifier *attr_id, int try_import) { - static PyObject *warnings_str = NULL; + PyObject *warnings_str; PyObject *warnings_module, *obj; + _Py_IDENTIFIER(warnings); + warnings_str = _PyUnicode_FromId(&PyId_warnings); if (warnings_str == NULL) { - warnings_str = PyUnicode_InternFromString("warnings"); - if (warnings_str == NULL) - return NULL; + return NULL; } /* don't try to import after the start of the Python finallization */ @@ -64,7 +64,7 @@ get_warnings_attr(const char *attr, int try_import) return NULL; } - obj = PyObject_GetAttrString(warnings_module, attr); + obj = _PyObject_GetAttrId(warnings_module, attr_id); Py_DECREF(warnings_module); if (obj == NULL && PyErr_ExceptionMatches(PyExc_AttributeError)) { PyErr_Clear(); @@ -77,8 +77,9 @@ static PyObject * get_once_registry(void) { PyObject *registry; + _Py_IDENTIFIER(onceregistry); - registry = get_warnings_attr("onceregistry", 0); + registry = get_warnings_attr(&PyId_onceregistry, 0); if (registry == NULL) { if (PyErr_Occurred()) return NULL; @@ -102,8 +103,9 @@ static PyObject * get_default_action(void) { PyObject *default_action; + _Py_IDENTIFIER(defaultaction); - default_action = get_warnings_attr("defaultaction", 0); + default_action = get_warnings_attr(&PyId_defaultaction, 0); if (default_action == NULL) { if (PyErr_Occurred()) { return NULL; @@ -132,8 +134,9 @@ get_filter(PyObject *category, PyObject *text, Py_ssize_t lineno, PyObject *action; Py_ssize_t i; PyObject *warnings_filters; + _Py_IDENTIFIER(filters); - warnings_filters = get_warnings_attr("filters", 0); + warnings_filters = get_warnings_attr(&PyId_filters, 0); if (warnings_filters == NULL) { if (PyErr_Occurred()) return NULL; @@ -389,11 +392,13 @@ call_show_warning(PyObject *category, PyObject *text, PyObject *message, PyObject *sourceline, PyObject *source) { PyObject *show_fn, *msg, *res, *warnmsg_cls = NULL; + _Py_IDENTIFIER(_showwarnmsg); + _Py_IDENTIFIER(WarningMessage); /* If the source parameter is set, try to get the Python implementation. The Python implementation is able to log the traceback where the source was allocated, whereas the C implementation doesn't. */ - show_fn = get_warnings_attr("_showwarnmsg", source != NULL); + show_fn = get_warnings_attr(&PyId__showwarnmsg, source != NULL); if (show_fn == NULL) { if (PyErr_Occurred()) return -1; @@ -407,7 +412,7 @@ call_show_warning(PyObject *category, PyObject *text, PyObject *message, goto error; } - warnmsg_cls = get_warnings_attr("WarningMessage", 0); + warnmsg_cls = get_warnings_attr(&PyId_WarningMessage, 0); if (warnmsg_cls == NULL) { if (!PyErr_Occurred()) { PyErr_SetString(PyExc_RuntimeError, @@ -1146,50 +1151,36 @@ static PyMethodDef warnings_functions[] = { static PyObject * create_filter(PyObject *category, const char *action) { - static PyObject *ignore_str = NULL; - static PyObject *error_str = NULL; - static PyObject *default_str = NULL; - static PyObject *always_str = NULL; - PyObject *action_obj = NULL; + _Py_IDENTIFIER(ignore); + _Py_IDENTIFIER(error); + _Py_IDENTIFIER(always); + _Py_static_string(PyId_default, "default"); + _Py_Identifier *id; if (!strcmp(action, "ignore")) { - if (ignore_str == NULL) { - ignore_str = PyUnicode_InternFromString("ignore"); - if (ignore_str == NULL) - return NULL; - } - action_obj = ignore_str; + id = &PyId_ignore; } else if (!strcmp(action, "error")) { - if (error_str == NULL) { - error_str = PyUnicode_InternFromString("error"); - if (error_str == NULL) - return NULL; - } - action_obj = error_str; + id = &PyId_error; } else if (!strcmp(action, "default")) { - if (default_str == NULL) { - default_str = PyUnicode_InternFromString("default"); - if (default_str == NULL) - return NULL; - } - action_obj = default_str; + id = &PyId_default; } else if (!strcmp(action, "always")) { - if (always_str == NULL) { - always_str = PyUnicode_InternFromString("always"); - if (always_str == NULL) - return NULL; - } - action_obj = always_str; + id = &PyId_always; } else { - Py_FatalError("unknown action"); + PyErr_SetString(PyExc_ValueError, "unknown action"); + return NULL; + } + + PyObject *action_str = _PyUnicode_FromId(id); + if (action_str == NULL) { + return NULL; } /* This assumes the line number is zero for now. */ - return PyTuple_Pack(5, action_obj, Py_None, + return PyTuple_Pack(5, action_str, Py_None, category, Py_None, _PyLong_Zero); } From d4341109746aa15e1909e63b30b93b6133ffe401 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 23 Nov 2017 00:12:09 +0100 Subject: [PATCH 07/75] bpo-32030: Add _PyCoreConfig.module_search_path_env (#4504) Changes: * Py_Main() initializes _PyCoreConfig.module_search_path_env from the PYTHONPATH environment variable. * PyInterpreterState_New() now initializes core_config and config fields * Compute sys.path a little bit ealier in _Py_InitializeMainInterpreter() and new_interpreter() * Add _Py_GetPathWithConfig() private function. --- Include/pylifecycle.h | 3 ++ Include/pystate.h | 9 ++++- Modules/getpath.c | 61 ++++++++++++++++++++++--------- Modules/main.c | 74 +++++++++++++++++++++++++++++++------ PC/getpathp.c | 27 ++++++++++---- Python/pylifecycle.c | 16 ++++---- Python/pystate.c | 85 +++++++++++++++++++++++-------------------- 7 files changed, 189 insertions(+), 86 deletions(-) diff --git a/Include/pylifecycle.h b/Include/pylifecycle.h index a75b77cc73372c..2a5e73a79cac42 100644 --- a/Include/pylifecycle.h +++ b/Include/pylifecycle.h @@ -93,6 +93,9 @@ PyAPI_FUNC(wchar_t *) Py_GetProgramFullPath(void); PyAPI_FUNC(wchar_t *) Py_GetPrefix(void); PyAPI_FUNC(wchar_t *) Py_GetExecPrefix(void); PyAPI_FUNC(wchar_t *) Py_GetPath(void); +#ifdef Py_BUILD_CORE +PyAPI_FUNC(wchar_t *) _Py_GetPathWithConfig(_PyCoreConfig *config); +#endif PyAPI_FUNC(void) Py_SetPath(const wchar_t *); #ifdef MS_WINDOWS int _Py_CheckPython3(); diff --git a/Include/pystate.h b/Include/pystate.h index a3840c96cb02ed..c5d3c3345f0050 100644 --- a/Include/pystate.h +++ b/Include/pystate.h @@ -30,6 +30,7 @@ typedef struct { unsigned long hash_seed; int _disable_importlib; /* Needed by freeze_importlib */ const char *allocator; /* Memory allocator: _PyMem_SetupAllocators() */ + wchar_t *module_search_path_env; /* PYTHONPATH environment variable */ int dev_mode; /* -X dev */ int faulthandler; /* -X faulthandler */ int tracemalloc; /* -X tracemalloc=N */ @@ -39,11 +40,13 @@ typedef struct { } _PyCoreConfig; #define _PyCoreConfig_INIT \ - {.ignore_environment = 0, \ + (_PyCoreConfig){\ + .ignore_environment = 0, \ .use_hash_seed = -1, \ .hash_seed = 0, \ ._disable_importlib = 0, \ .allocator = NULL, \ + .module_search_path_env = NULL, \ .dev_mode = 0, \ .faulthandler = 0, \ .tracemalloc = 0, \ @@ -61,7 +64,9 @@ typedef struct { int install_signal_handlers; } _PyMainInterpreterConfig; -#define _PyMainInterpreterConfig_INIT {-1} +#define _PyMainInterpreterConfig_INIT \ + (_PyMainInterpreterConfig){\ + .install_signal_handlers = -1} typedef struct _is { diff --git a/Modules/getpath.c b/Modules/getpath.c index dd3387a9d77f98..ad4a4e5e718d03 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -456,14 +456,12 @@ search_for_exec_prefix(wchar_t *argv0_path, wchar_t *home, } static void -calculate_path(void) +calculate_path(_PyCoreConfig *core_config) { extern wchar_t *Py_GetProgramName(void); static const wchar_t delimiter[2] = {DELIM, '\0'}; static const wchar_t separator[2] = {SEP, '\0'}; - char *_rtpypath = Py_GETENV("PYTHONPATH"); /* XXX use wide version on Windows */ - wchar_t *rtpypath = NULL; wchar_t *home = Py_GetPythonHome(); char *_path = getenv("PATH"); wchar_t *path_buffer = NULL; @@ -707,11 +705,22 @@ calculate_path(void) */ bufsz = 0; - if (_rtpypath && _rtpypath[0] != '\0') { - size_t rtpypath_len; - rtpypath = Py_DecodeLocale(_rtpypath, &rtpypath_len); - if (rtpypath != NULL) - bufsz += rtpypath_len + 1; + wchar_t *env_path = NULL; + if (core_config) { + if (core_config->module_search_path_env) { + bufsz += wcslen(core_config->module_search_path_env) + 1; + } + } + else { + char *env_pathb = Py_GETENV("PYTHONPATH"); + if (env_pathb && env_pathb[0] != '\0') { + size_t env_path_len; + env_path = Py_DecodeLocale(env_pathb, &env_path_len); + /* FIXME: handle decoding and memory error */ + if (env_path != NULL) { + bufsz += env_path_len + 1; + } + } } defpath = _pythonpath; @@ -742,12 +751,20 @@ calculate_path(void) } /* Run-time value of $PYTHONPATH goes first */ - if (rtpypath) { - wcscpy(buf, rtpypath); - wcscat(buf, delimiter); + buf[0] = '\0'; + if (core_config) { + if (core_config->module_search_path_env) { + wcscpy(buf, core_config->module_search_path_env); + wcscat(buf, delimiter); + } } - else - buf[0] = '\0'; + else { + if (env_path) { + wcscpy(buf, env_path); + wcscat(buf, delimiter); + } + } + PyMem_RawFree(env_path); /* Next is the default zip path */ wcscat(buf, zip_path); @@ -818,7 +835,6 @@ calculate_path(void) PyMem_RawFree(_prefix); PyMem_RawFree(_exec_prefix); PyMem_RawFree(lib_python); - PyMem_RawFree(rtpypath); } @@ -841,11 +857,20 @@ Py_SetPath(const wchar_t *path) } } +wchar_t * +_Py_GetPathWithConfig(_PyCoreConfig *core_config) +{ + if (!module_search_path) { + calculate_path(core_config); + } + return module_search_path; +} + wchar_t * Py_GetPath(void) { if (!module_search_path) - calculate_path(); + calculate_path(NULL); return module_search_path; } @@ -853,7 +878,7 @@ wchar_t * Py_GetPrefix(void) { if (!module_search_path) - calculate_path(); + calculate_path(NULL); return prefix; } @@ -861,7 +886,7 @@ wchar_t * Py_GetExecPrefix(void) { if (!module_search_path) - calculate_path(); + calculate_path(NULL); return exec_prefix; } @@ -869,7 +894,7 @@ wchar_t * Py_GetProgramFullPath(void) { if (!module_search_path) - calculate_path(); + calculate_path(NULL); return progpath; } diff --git a/Modules/main.c b/Modules/main.c index e5e4f33f51c072..3bd93e37601931 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -399,6 +399,7 @@ typedef struct { _PyInitError err; /* PYTHONWARNINGS env var */ _Py_OptList env_warning_options; + /* PYTHONPATH env var */ int argc; wchar_t **argv; } _PyMain; @@ -441,6 +442,8 @@ pymain_free_impl(_PyMain *pymain) Py_CLEAR(pymain->main_importer_path); PyMem_RawFree(pymain->program_name); + PyMem_RawFree(pymain->core_config.module_search_path_env); + #ifdef __INSURE__ /* Insure++ is a memory analysis tool that aids in discovering * memory leaks and other memory problems. On Python exit, the @@ -502,15 +505,22 @@ pymain_run_main_from_importer(_PyMain *pymain) static wchar_t* -pymain_strdup(_PyMain *pymain, wchar_t *str) +pymain_wstrdup(_PyMain *pymain, wchar_t *str) { size_t len = wcslen(str) + 1; /* +1 for NUL character */ - wchar_t *str2 = PyMem_RawMalloc(sizeof(wchar_t) * len); + if (len > (size_t)PY_SSIZE_T_MAX / sizeof(wchar_t)) { + pymain->err = INIT_NO_MEMORY(); + return NULL; + } + + size_t size = len * sizeof(wchar_t); + wchar_t *str2 = PyMem_RawMalloc(size); if (str2 == NULL) { pymain->err = INIT_NO_MEMORY(); return NULL; } - memcpy(str2, str, len * sizeof(wchar_t)); + + memcpy(str2, str, size); return str2; } @@ -518,7 +528,7 @@ pymain_strdup(_PyMain *pymain, wchar_t *str) static int pymain_optlist_append(_PyMain *pymain, _Py_OptList *list, wchar_t *str) { - wchar_t *str2 = pymain_strdup(pymain, str); + wchar_t *str2 = pymain_wstrdup(pymain, str); if (str2 == NULL) { return -1; } @@ -762,14 +772,12 @@ pymain_warnings_envvar(_PyMain *pymain) wchar_t *wp; if ((wp = _wgetenv(L"PYTHONWARNINGS")) && *wp != L'\0') { - wchar_t *buf, *warning, *context = NULL; + wchar_t *warning, *context = NULL; - buf = (wchar_t *)PyMem_RawMalloc((wcslen(wp) + 1) * sizeof(wchar_t)); + wchar_t *buf = pymain_wstrdup(pymain, wp); if (buf == NULL) { - pymain->err = INIT_NO_MEMORY(); return -1; } - wcscpy(buf, wp); for (warning = wcstok_s(buf, L",", &context); warning != NULL; warning = wcstok_s(NULL, L",", &context)) { @@ -805,12 +813,11 @@ pymain_warnings_envvar(_PyMain *pymain) if (len == (size_t)-2) { pymain->err = _Py_INIT_ERR("failed to decode " "PYTHONWARNINGS"); - return -1; } else { pymain->err = INIT_NO_MEMORY(); - return -1; } + return -1; } if (pymain_optlist_append(pymain, &pymain->env_warning_options, warning) < 0) { @@ -929,7 +936,7 @@ pymain_get_program_name(_PyMain *pymain) if (pymain->program_name == NULL) { /* Use argv[0] by default */ - pymain->program_name = pymain_strdup(pymain, pymain->argv[0]); + pymain->program_name = pymain_wstrdup(pymain, pymain->argv[0]); if (pymain->program_name == NULL) { return -1; } @@ -1362,6 +1369,48 @@ pymain_set_flags_from_env(_PyMain *pymain) } +static int +pymain_init_pythonpath(_PyMain *pymain) +{ + if (Py_IgnoreEnvironmentFlag) { + return 0; + } + +#ifdef MS_WINDOWS + wchar_t *path = _wgetenv(L"PYTHONPATH"); + if (!path || path[0] == '\0') { + return 0; + } + + wchar_t *path2 = pymain_wstrdup(pymain, path); + if (path2 == NULL) { + return -1; + } + + pymain->core_config.module_search_path_env = path2; +#else + char *path = pymain_get_env_var("PYTHONPATH"); + if (!path) { + return 0; + } + + size_t len; + wchar_t *wpath = Py_DecodeLocale(path, &len); + if (!wpath) { + if (len == (size_t)-2) { + pymain->err = _Py_INIT_ERR("failed to decode PYTHONHOME"); + } + else { + pymain->err = INIT_NO_MEMORY(); + } + return -1; + } + pymain->core_config.module_search_path_env = wpath; +#endif + return 0; +} + + static int pymain_parse_envvars(_PyMain *pymain) { @@ -1383,6 +1432,9 @@ pymain_parse_envvars(_PyMain *pymain) return -1; } core_config->allocator = Py_GETENV("PYTHONMALLOC"); + if (pymain_init_pythonpath(pymain) < 0) { + return -1; + } /* -X options */ if (pymain_get_xoption(pymain, L"showrefcount")) { diff --git a/PC/getpathp.c b/PC/getpathp.c index 9bbc7bf0b27ba5..b182ae6a58df68 100644 --- a/PC/getpathp.c +++ b/PC/getpathp.c @@ -624,7 +624,7 @@ read_pth_file(const wchar_t *path, wchar_t *prefix, int *isolated, int *nosite) static void -calculate_path(void) +calculate_path(_PyCoreConfig *core_config) { wchar_t argv0_path[MAXPATHLEN+1]; wchar_t *buf; @@ -637,8 +637,13 @@ calculate_path(void) wchar_t *userpath = NULL; wchar_t zip_path[MAXPATHLEN+1]; - if (!Py_IgnoreEnvironmentFlag) { + if (core_config) { + envpath = core_config->module_search_path_env; + } + else if (!Py_IgnoreEnvironmentFlag) { envpath = _wgetenv(L"PYTHONPATH"); + if (envpath && *envpath == '\0') + envpath = NULL; } get_progpath(); @@ -709,9 +714,6 @@ calculate_path(void) else wcscpy_s(prefix, MAXPATHLEN+1, pythonhome); - if (envpath && *envpath == '\0') - envpath = NULL; - skiphome = pythonhome==NULL ? 0 : 1; #ifdef Py_ENABLE_SHARED @@ -896,11 +898,20 @@ Py_SetPath(const wchar_t *path) } } +wchar_t * +_Py_GetPathWithConfig(_PyCoreConfig *core_config) +{ + if (!module_search_path) { + calculate_path(core_config); + } + return module_search_path; +} + wchar_t * Py_GetPath(void) { if (!module_search_path) - calculate_path(); + calculate_path(NULL); return module_search_path; } @@ -908,7 +919,7 @@ wchar_t * Py_GetPrefix(void) { if (!module_search_path) - calculate_path(); + calculate_path(NULL); return prefix; } @@ -922,7 +933,7 @@ wchar_t * Py_GetProgramFullPath(void) { if (!module_search_path) - calculate_path(); + calculate_path(NULL); return progpath; } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 36fcf61c33d2d3..9eeac9d33be601 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -48,8 +48,6 @@ _Py_IDENTIFIER(threading); extern "C" { #endif -extern wchar_t *Py_GetPath(void); - extern grammar _PyParser_Grammar; /* From graminit.c */ /* Forward */ @@ -842,6 +840,11 @@ _Py_InitializeMainInterpreter(const _PyMainInterpreterConfig *config) /* Now finish configuring the main interpreter */ interp->config = *config; + /* GetPath may initialize state that _PySys_EndInit locks + in, and so has to be called first. */ + /* TODO: Call Py_GetPath() in Py_ReadConfig, rather than here */ + wchar_t *sys_path = _Py_GetPathWithConfig(&interp->core_config); + if (interp->core_config._disable_importlib) { /* Special mode for freeze_importlib: run with no import system * @@ -857,10 +860,7 @@ _Py_InitializeMainInterpreter(const _PyMainInterpreterConfig *config) return _Py_INIT_ERR("can't initialize time"); /* Finish setting up the sys module and import system */ - /* GetPath may initialize state that _PySys_EndInit locks - in, and so has to be called first. */ - /* TODO: Call Py_GetPath() in Py_ReadConfig, rather than here */ - PySys_SetPath(Py_GetPath()); + PySys_SetPath(sys_path); if (_PySys_EndInit(interp->sysdict) < 0) return _Py_INIT_ERR("can't finish initializing sys"); @@ -1301,6 +1301,8 @@ new_interpreter(PyThreadState **tstate_p) /* XXX The following is lax in error checking */ + wchar_t *sys_path = _Py_GetPathWithConfig(&interp->core_config); + PyObject *modules = PyDict_New(); if (modules == NULL) { return _Py_INIT_ERR("can't make modules dictionary"); @@ -1314,7 +1316,7 @@ new_interpreter(PyThreadState **tstate_p) goto handle_error; Py_INCREF(interp->sysdict); PyDict_SetItemString(interp->sysdict, "modules", modules); - PySys_SetPath(Py_GetPath()); + PySys_SetPath(sys_path); _PySys_EndInit(interp->sysdict); } diff --git a/Python/pystate.c b/Python/pystate.c index 807ac4eb9d18bc..f6fbb4d041ea4b 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -106,55 +106,60 @@ PyInterpreterState_New(void) PyInterpreterState *interp = (PyInterpreterState *) PyMem_RawMalloc(sizeof(PyInterpreterState)); - if (interp != NULL) { - interp->modules = NULL; - interp->modules_by_index = NULL; - interp->sysdict = NULL; - interp->builtins = NULL; - interp->builtins_copy = NULL; - interp->tstate_head = NULL; - interp->check_interval = 100; - interp->num_threads = 0; - interp->pythread_stacksize = 0; - interp->codec_search_path = NULL; - interp->codec_search_cache = NULL; - interp->codec_error_registry = NULL; - interp->codecs_initialized = 0; - interp->fscodec_initialized = 0; - interp->importlib = NULL; - interp->import_func = NULL; - interp->eval_frame = _PyEval_EvalFrameDefault; - interp->co_extra_user_count = 0; + if (interp == NULL) { + return NULL; + } + + + interp->modules = NULL; + interp->modules_by_index = NULL; + interp->sysdict = NULL; + interp->builtins = NULL; + interp->builtins_copy = NULL; + interp->tstate_head = NULL; + interp->check_interval = 100; + interp->num_threads = 0; + interp->pythread_stacksize = 0; + interp->codec_search_path = NULL; + interp->codec_search_cache = NULL; + interp->codec_error_registry = NULL; + interp->codecs_initialized = 0; + interp->fscodec_initialized = 0; + interp->core_config = _PyCoreConfig_INIT; + interp->config = _PyMainInterpreterConfig_INIT; + interp->importlib = NULL; + interp->import_func = NULL; + interp->eval_frame = _PyEval_EvalFrameDefault; + interp->co_extra_user_count = 0; #ifdef HAVE_DLOPEN #if HAVE_DECL_RTLD_NOW - interp->dlopenflags = RTLD_NOW; + interp->dlopenflags = RTLD_NOW; #else - interp->dlopenflags = RTLD_LAZY; + interp->dlopenflags = RTLD_LAZY; #endif #endif #ifdef HAVE_FORK - interp->before_forkers = NULL; - interp->after_forkers_parent = NULL; - interp->after_forkers_child = NULL; + interp->before_forkers = NULL; + interp->after_forkers_parent = NULL; + interp->after_forkers_child = NULL; #endif - HEAD_LOCK(); - interp->next = _PyRuntime.interpreters.head; - if (_PyRuntime.interpreters.main == NULL) { - _PyRuntime.interpreters.main = interp; - } - _PyRuntime.interpreters.head = interp; - if (_PyRuntime.interpreters.next_id < 0) { - /* overflow or Py_Initialize() not called! */ - PyErr_SetString(PyExc_RuntimeError, - "failed to get an interpreter ID"); - interp = NULL; - } else { - interp->id = _PyRuntime.interpreters.next_id; - _PyRuntime.interpreters.next_id += 1; - } - HEAD_UNLOCK(); + HEAD_LOCK(); + interp->next = _PyRuntime.interpreters.head; + if (_PyRuntime.interpreters.main == NULL) { + _PyRuntime.interpreters.main = interp; + } + _PyRuntime.interpreters.head = interp; + if (_PyRuntime.interpreters.next_id < 0) { + /* overflow or Py_Initialize() not called! */ + PyErr_SetString(PyExc_RuntimeError, + "failed to get an interpreter ID"); + interp = NULL; + } else { + interp->id = _PyRuntime.interpreters.next_id; + _PyRuntime.interpreters.next_id += 1; } + HEAD_UNLOCK(); return interp; } From 20d48a44a54ed5e4a6df00e89ae27e3983128265 Mon Sep 17 00:00:00 2001 From: Cheryl Sabella Date: Wed, 22 Nov 2017 19:05:25 -0500 Subject: [PATCH 08/75] bpo-32100: IDLE: Fix pathbrowser errors; improve tests. (#4484) Patch mostly by Cheryl Sabella --- Lib/idlelib/browser.py | 12 ++-- Lib/idlelib/editor.py | 2 +- Lib/idlelib/idle_test/test_browser.py | 26 +++---- Lib/idlelib/idle_test/test_pathbrowser.py | 67 ++++++++++++++++++- Lib/idlelib/pathbrowser.py | 8 +-- .../2017-11-21-08-26-08.bpo-32100.P43qx2.rst | 2 + 6 files changed, 90 insertions(+), 27 deletions(-) create mode 100644 Misc/NEWS.d/next/IDLE/2017-11-21-08-26-08.bpo-32100.P43qx2.rst diff --git a/Lib/idlelib/browser.py b/Lib/idlelib/browser.py index 79eaeb7eb45bf4..447dafcc515e6c 100644 --- a/Lib/idlelib/browser.py +++ b/Lib/idlelib/browser.py @@ -79,9 +79,6 @@ def __init__(self, master, path, *, _htest=False, _utest=False): creating ModuleBrowserTreeItem as the rootnode for the tree and subsequently in the children. """ - global file_open - if not (_htest or _utest): - file_open = pyshell.flist.open self.master = master self.path = path self._htest = _htest @@ -95,9 +92,13 @@ def close(self, event=None): def init(self): "Create browser tkinter widgets, including the tree." + global file_open root = self.master - # reset pyclbr + flist = (pyshell.flist if not (self._htest or self._utest) + else pyshell.PyShellFileList(root)) + file_open = flist.open pyclbr._modules.clear() + # create top self.top = top = ListedToplevel(root) top.protocol("WM_DELETE_WINDOW", self.close) @@ -107,6 +108,7 @@ def init(self): (root.winfo_rootx(), root.winfo_rooty() + 200)) self.settitle() top.focus_set() + # create scrolled canvas theme = idleConf.CurrentTheme() background = idleConf.GetHighlight(theme, 'normal')['background'] @@ -236,8 +238,6 @@ class Nested_in_func(TreeNode): def nested_in_class(): pass def closure(): class Nested_in_closure: pass - global file_open - file_open = pyshell.PyShellFileList(parent).open ModuleBrowser(parent, file, _htest=True) if __name__ == "__main__": diff --git a/Lib/idlelib/editor.py b/Lib/idlelib/editor.py index 68450c921f2fad..b51c45c97e50f2 100644 --- a/Lib/idlelib/editor.py +++ b/Lib/idlelib/editor.py @@ -668,7 +668,7 @@ def open_module_browser(self, event=None): def open_path_browser(self, event=None): from idlelib import pathbrowser - pathbrowser.PathBrowser(self.flist) + pathbrowser.PathBrowser(self.root) return "break" def open_turtle_demo(self, event = None): diff --git a/Lib/idlelib/idle_test/test_browser.py b/Lib/idlelib/idle_test/test_browser.py index 59e03c5aab3c9e..34eb332c1df434 100644 --- a/Lib/idlelib/idle_test/test_browser.py +++ b/Lib/idlelib/idle_test/test_browser.py @@ -4,17 +4,19 @@ (Higher, because should exclude 3 lines that .coveragerc won't exclude.) """ +from collections import deque import os.path -import unittest import pyclbr +from tkinter import Tk -from idlelib import browser, filelist -from idlelib.tree import TreeNode from test.support import requires +import unittest from unittest import mock -from tkinter import Tk from idlelib.idle_test.mock_idle import Func -from collections import deque + +from idlelib import browser +from idlelib import filelist +from idlelib.tree import TreeNode class ModuleBrowserTest(unittest.TestCase): @@ -29,6 +31,7 @@ def setUpClass(cls): @classmethod def tearDownClass(cls): cls.mb.close() + cls.root.update_idletasks() cls.root.destroy() del cls.root, cls.mb @@ -38,6 +41,7 @@ def test_init(self): eq(mb.path, __file__) eq(pyclbr._modules, {}) self.assertIsInstance(mb.node, TreeNode) + self.assertIsNotNone(browser.file_open) def test_settitle(self): mb = self.mb @@ -151,10 +155,9 @@ def test_getsublist(self): self.assertEqual(sub0.name, 'f0') self.assertEqual(sub1.name, 'C0(base)') - - def test_ondoubleclick(self): + @mock.patch('idlelib.browser.file_open') + def test_ondoubleclick(self, fopen): mbt = self.mbt - fopen = browser.file_open = mock.Mock() with mock.patch('os.path.exists', return_value=False): mbt.OnDoubleClick() @@ -165,8 +168,6 @@ def test_ondoubleclick(self): fopen.assert_called() fopen.called_with(fname) - del browser.file_open - class ChildBrowserTreeItemTest(unittest.TestCase): @@ -212,14 +213,13 @@ def test_getsublist(self): eq(self.cbt_F1.GetSubList(), []) - def test_ondoubleclick(self): - fopen = browser.file_open = mock.Mock() + @mock.patch('idlelib.browser.file_open') + def test_ondoubleclick(self, fopen): goto = fopen.return_value.gotoline = mock.Mock() self.cbt_F1.OnDoubleClick() fopen.assert_called() goto.assert_called() goto.assert_called_with(self.cbt_F1.obj.lineno) - del browser.file_open # Failure test would have to raise OSError or AttributeError. diff --git a/Lib/idlelib/idle_test/test_pathbrowser.py b/Lib/idlelib/idle_test/test_pathbrowser.py index 813cbcc63167cc..74b716a3199327 100644 --- a/Lib/idlelib/idle_test/test_pathbrowser.py +++ b/Lib/idlelib/idle_test/test_pathbrowser.py @@ -1,11 +1,68 @@ +""" Test idlelib.pathbrowser. +""" + + +import os.path +import pyclbr # for _modules +import sys # for sys.path +from tkinter import Tk + +from test.support import requires import unittest -import os -import sys -import idlelib +from idlelib.idle_test.mock_idle import Func + +import idlelib # for __file__ +from idlelib import browser from idlelib import pathbrowser +from idlelib.tree import TreeNode + class PathBrowserTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + requires('gui') + cls.root = Tk() + cls.root.withdraw() + cls.pb = pathbrowser.PathBrowser(cls.root, _utest=True) + + @classmethod + def tearDownClass(cls): + cls.pb.close() + cls.root.update_idletasks() + cls.root.destroy() + del cls.root, cls.pb + + def test_init(self): + pb = self.pb + eq = self.assertEqual + eq(pb.master, self.root) + eq(pyclbr._modules, {}) + self.assertIsInstance(pb.node, TreeNode) + self.assertIsNotNone(browser.file_open) + + def test_settitle(self): + pb = self.pb + self.assertEqual(pb.top.title(), 'Path Browser') + self.assertEqual(pb.top.iconname(), 'Path Browser') + + def test_rootnode(self): + pb = self.pb + rn = pb.rootnode() + self.assertIsInstance(rn, pathbrowser.PathBrowserTreeItem) + + def test_close(self): + pb = self.pb + pb.top.destroy = Func() + pb.node.destroy = Func() + pb.close() + self.assertTrue(pb.top.destroy.called) + self.assertTrue(pb.node.destroy.called) + del pb.top.destroy, pb.node.destroy + + +class DirBrowserTreeItemTest(unittest.TestCase): + def test_DirBrowserTreeItem(self): # Issue16226 - make sure that getting a sublist works d = pathbrowser.DirBrowserTreeItem('') @@ -16,6 +73,9 @@ def test_DirBrowserTreeItem(self): self.assertEqual(d.ispackagedir(dir), True) self.assertEqual(d.ispackagedir(dir + '/Icons'), False) + +class PathBrowserTreeItemTest(unittest.TestCase): + def test_PathBrowserTreeItem(self): p = pathbrowser.PathBrowserTreeItem() self.assertEqual(p.GetText(), 'sys.path') @@ -23,5 +83,6 @@ def test_PathBrowserTreeItem(self): self.assertEqual(len(sub), len(sys.path)) self.assertEqual(type(sub[0]), pathbrowser.DirBrowserTreeItem) + if __name__ == '__main__': unittest.main(verbosity=2, exit=False) diff --git a/Lib/idlelib/pathbrowser.py b/Lib/idlelib/pathbrowser.py index c0aa2a1590916c..c61ae0f4fb96cc 100644 --- a/Lib/idlelib/pathbrowser.py +++ b/Lib/idlelib/pathbrowser.py @@ -9,13 +9,14 @@ class PathBrowser(ModuleBrowser): - def __init__(self, flist, *, _htest=False, _utest=False): + def __init__(self, master, *, _htest=False, _utest=False): """ _htest - bool, change box location when running htest """ + self.master = master self._htest = _htest self._utest = _utest - self.init(flist) + self.init() def settitle(self): "Set window titles." @@ -100,8 +101,7 @@ def listmodules(self, allnames): def _path_browser(parent): # htest # - flist = PyShellFileList(parent) - PathBrowser(flist, _htest=True) + PathBrowser(parent, _htest=True) parent.mainloop() if __name__ == "__main__": diff --git a/Misc/NEWS.d/next/IDLE/2017-11-21-08-26-08.bpo-32100.P43qx2.rst b/Misc/NEWS.d/next/IDLE/2017-11-21-08-26-08.bpo-32100.P43qx2.rst new file mode 100644 index 00000000000000..c5ee6736a84565 --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2017-11-21-08-26-08.bpo-32100.P43qx2.rst @@ -0,0 +1,2 @@ +IDLE: Fix old and new bugs in pathbrowser; improve tests. +Patch mostly by Cheryl Sabella. From 0784a2e5b174d2dbf7b144d480559e650c5cf64c Mon Sep 17 00:00:00 2001 From: Jesse-Bakker Date: Thu, 23 Nov 2017 01:23:28 +0100 Subject: [PATCH 09/75] bpo-10049: Add a "no-op" (null) context manager to contextlib (GH-4464) Adds a simpler and faster alternative to ExitStack for handling single optional context managers without having to change the lexical structure of your code. --- Doc/library/contextlib.rst | 40 ++++++++++--------- Lib/contextlib.py | 23 ++++++++++- Lib/test/test_contextlib.py | 10 +++++ .../2017-11-22-17-21-01.bpo-10049.ttsBqb.rst | 3 ++ 4 files changed, 57 insertions(+), 19 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2017-11-22-17-21-01.bpo-10049.ttsBqb.rst diff --git a/Doc/library/contextlib.rst b/Doc/library/contextlib.rst index 19793693b7ba68..48ca0da6b95f3a 100644 --- a/Doc/library/contextlib.rst +++ b/Doc/library/contextlib.rst @@ -137,6 +137,28 @@ Functions and classes provided: ``page.close()`` will be called when the :keyword:`with` block is exited. +.. _simplifying-support-for-single-optional-context-managers: + +.. function:: nullcontext(enter_result=None) + + Return a context manager that returns enter_result from ``__enter__``, but + otherwise does nothing. It is intended to be used as a stand-in for an + optional context manager, for example:: + + def process_file(file_or_path): + if isinstance(file_or_path, str): + # If string, open file + cm = open(file_or_path) + else: + # Caller is responsible for closing file + cm = nullcontext(file_or_path) + + with cm as file: + # Perform processing on the file + + .. versionadded:: 3.7 + + .. function:: suppress(*exceptions) Return a context manager that suppresses any of the specified exceptions @@ -433,24 +455,6 @@ statements to manage arbitrary resources that don't natively support the context management protocol. -Simplifying support for single optional context managers -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -In the specific case of a single optional context manager, :class:`ExitStack` -instances can be used as a "do nothing" context manager, allowing a context -manager to easily be omitted without affecting the overall structure of -the source code:: - - def debug_trace(details): - if __debug__: - return TraceContext(details) - # Don't do anything special with the context in release mode - return ExitStack() - - with debug_trace(): - # Suite is traced in debug mode, but runs normally otherwise - - Catching exceptions from ``__enter__`` methods ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/Lib/contextlib.py b/Lib/contextlib.py index 962cedab490eb2..c1f8a84617fce4 100644 --- a/Lib/contextlib.py +++ b/Lib/contextlib.py @@ -5,7 +5,7 @@ from collections import deque from functools import wraps -__all__ = ["asynccontextmanager", "contextmanager", "closing", +__all__ = ["asynccontextmanager", "contextmanager", "closing", "nullcontext", "AbstractContextManager", "ContextDecorator", "ExitStack", "redirect_stdout", "redirect_stderr", "suppress"] @@ -469,3 +469,24 @@ def _fix_exception_context(new_exc, old_exc): exc_details[1].__context__ = fixed_ctx raise return received_exc and suppressed_exc + + +class nullcontext(AbstractContextManager): + """Context manager that does no additional processing. + + Used as a stand-in for a normal context manager, when a particular + block of code is only sometimes used with a normal context manager: + + cm = optional_cm if condition else nullcontext() + with cm: + # Perform operation, using optional_cm if condition is True + """ + + def __init__(self, enter_result=None): + self.enter_result = enter_result + + def __enter__(self): + return self.enter_result + + def __exit__(self, *excinfo): + pass diff --git a/Lib/test/test_contextlib.py b/Lib/test/test_contextlib.py index 64b6578ff94eb3..1a5e6edad9b28f 100644 --- a/Lib/test/test_contextlib.py +++ b/Lib/test/test_contextlib.py @@ -252,6 +252,16 @@ def close(self): 1 / 0 self.assertEqual(state, [1]) + +class NullcontextTestCase(unittest.TestCase): + def test_nullcontext(self): + class C: + pass + c = C() + with nullcontext(c) as c_in: + self.assertIs(c_in, c) + + class FileContextTestCase(unittest.TestCase): def testWithOpen(self): diff --git a/Misc/NEWS.d/next/Library/2017-11-22-17-21-01.bpo-10049.ttsBqb.rst b/Misc/NEWS.d/next/Library/2017-11-22-17-21-01.bpo-10049.ttsBqb.rst new file mode 100644 index 00000000000000..b6153c235d0cc4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-11-22-17-21-01.bpo-10049.ttsBqb.rst @@ -0,0 +1,3 @@ +Added *nullcontext* no-op context manager to contextlib. This provides a +simpler and faster alternative to ExitStack() when handling optional context +managers. From e32e79f7d8216b78ac9e61bb1f2eee693108d4ee Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 23 Nov 2017 01:49:45 +0100 Subject: [PATCH 10/75] bpo-32030: Move PYTHONPATH to _PyMainInterpreterConfig (#4511) Move _PyCoreConfig.module_search_path_env to _PyMainInterpreterConfig structure. --- Include/pylifecycle.h | 2 +- Include/pystate.h | 6 +++--- Modules/getpath.c | 18 +++++++++--------- Modules/main.c | 18 +++++++++--------- PC/getpathp.c | 10 +++++----- Python/pylifecycle.c | 4 ++-- 6 files changed, 29 insertions(+), 29 deletions(-) diff --git a/Include/pylifecycle.h b/Include/pylifecycle.h index 2a5e73a79cac42..5eaa74edab8f98 100644 --- a/Include/pylifecycle.h +++ b/Include/pylifecycle.h @@ -94,7 +94,7 @@ PyAPI_FUNC(wchar_t *) Py_GetPrefix(void); PyAPI_FUNC(wchar_t *) Py_GetExecPrefix(void); PyAPI_FUNC(wchar_t *) Py_GetPath(void); #ifdef Py_BUILD_CORE -PyAPI_FUNC(wchar_t *) _Py_GetPathWithConfig(_PyCoreConfig *config); +PyAPI_FUNC(wchar_t *) _Py_GetPathWithConfig(_PyMainInterpreterConfig *config); #endif PyAPI_FUNC(void) Py_SetPath(const wchar_t *); #ifdef MS_WINDOWS diff --git a/Include/pystate.h b/Include/pystate.h index c5d3c3345f0050..b2739f1db25390 100644 --- a/Include/pystate.h +++ b/Include/pystate.h @@ -30,7 +30,6 @@ typedef struct { unsigned long hash_seed; int _disable_importlib; /* Needed by freeze_importlib */ const char *allocator; /* Memory allocator: _PyMem_SetupAllocators() */ - wchar_t *module_search_path_env; /* PYTHONPATH environment variable */ int dev_mode; /* -X dev */ int faulthandler; /* -X faulthandler */ int tracemalloc; /* -X tracemalloc=N */ @@ -46,7 +45,6 @@ typedef struct { .hash_seed = 0, \ ._disable_importlib = 0, \ .allocator = NULL, \ - .module_search_path_env = NULL, \ .dev_mode = 0, \ .faulthandler = 0, \ .tracemalloc = 0, \ @@ -62,11 +60,13 @@ typedef struct { */ typedef struct { int install_signal_handlers; + wchar_t *module_search_path_env; /* PYTHONPATH environment variable */ } _PyMainInterpreterConfig; #define _PyMainInterpreterConfig_INIT \ (_PyMainInterpreterConfig){\ - .install_signal_handlers = -1} + .install_signal_handlers = -1, \ + .module_search_path_env = NULL} typedef struct _is { diff --git a/Modules/getpath.c b/Modules/getpath.c index ad4a4e5e718d03..ead143280b74e1 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -456,7 +456,7 @@ search_for_exec_prefix(wchar_t *argv0_path, wchar_t *home, } static void -calculate_path(_PyCoreConfig *core_config) +calculate_path(_PyMainInterpreterConfig *config) { extern wchar_t *Py_GetProgramName(void); @@ -706,9 +706,9 @@ calculate_path(_PyCoreConfig *core_config) bufsz = 0; wchar_t *env_path = NULL; - if (core_config) { - if (core_config->module_search_path_env) { - bufsz += wcslen(core_config->module_search_path_env) + 1; + if (config) { + if (config->module_search_path_env) { + bufsz += wcslen(config->module_search_path_env) + 1; } } else { @@ -752,9 +752,9 @@ calculate_path(_PyCoreConfig *core_config) /* Run-time value of $PYTHONPATH goes first */ buf[0] = '\0'; - if (core_config) { - if (core_config->module_search_path_env) { - wcscpy(buf, core_config->module_search_path_env); + if (config) { + if (config->module_search_path_env) { + wcscpy(buf, config->module_search_path_env); wcscat(buf, delimiter); } } @@ -858,10 +858,10 @@ Py_SetPath(const wchar_t *path) } wchar_t * -_Py_GetPathWithConfig(_PyCoreConfig *core_config) +_Py_GetPathWithConfig(_PyMainInterpreterConfig *config) { if (!module_search_path) { - calculate_path(core_config); + calculate_path(config); } return module_search_path; } diff --git a/Modules/main.c b/Modules/main.c index 3bd93e37601931..8390af292383ce 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -389,6 +389,7 @@ typedef struct { /* non-zero is stdin is a TTY or if -i option is used */ int stdin_is_interactive; _PyCoreConfig core_config; + _PyMainInterpreterConfig config; _Py_CommandLineDetails cmdline; PyObject *main_importer_path; /* non-zero if filename, command (-c) or module (-m) is set @@ -409,6 +410,7 @@ typedef struct { {.status = 0, \ .cf = {.cf_flags = 0}, \ .core_config = _PyCoreConfig_INIT, \ + .config = _PyMainInterpreterConfig_INIT, \ .main_importer_path = NULL, \ .run_code = -1, \ .program_name = NULL, \ @@ -442,7 +444,7 @@ pymain_free_impl(_PyMain *pymain) Py_CLEAR(pymain->main_importer_path); PyMem_RawFree(pymain->program_name); - PyMem_RawFree(pymain->core_config.module_search_path_env); + PyMem_RawFree(pymain->config.module_search_path_env); #ifdef __INSURE__ /* Insure++ is a memory analysis tool that aids in discovering @@ -957,20 +959,16 @@ pymain_get_program_name(_PyMain *pymain) static int pymain_init_main_interpreter(_PyMain *pymain) { - _PyMainInterpreterConfig config = _PyMainInterpreterConfig_INIT; _PyInitError err; - /* TODO: Moar config options! */ - config.install_signal_handlers = 1; - /* TODO: Print any exceptions raised by these operations */ - err = _Py_ReadMainInterpreterConfig(&config); + err = _Py_ReadMainInterpreterConfig(&pymain->config); if (_Py_INIT_FAILED(err)) { pymain->err = err; return -1; } - err = _Py_InitializeMainInterpreter(&config); + err = _Py_InitializeMainInterpreter(&pymain->config); if (_Py_INIT_FAILED(err)) { pymain->err = err; return -1; @@ -1387,7 +1385,7 @@ pymain_init_pythonpath(_PyMain *pymain) return -1; } - pymain->core_config.module_search_path_env = path2; + pymain->config.module_search_path_env = path2; #else char *path = pymain_get_env_var("PYTHONPATH"); if (!path) { @@ -1405,7 +1403,7 @@ pymain_init_pythonpath(_PyMain *pymain) } return -1; } - pymain->core_config.module_search_path_env = wpath; + pymain->config.module_search_path_env = wpath; #endif return 0; } @@ -1574,6 +1572,8 @@ pymain_init(_PyMain *pymain) } pymain->core_config._disable_importlib = 0; + /* TODO: Moar config options! */ + pymain->config.install_signal_handlers = 1; orig_argc = pymain->argc; /* For Py_GetArgcArgv() */ orig_argv = pymain->argv; diff --git a/PC/getpathp.c b/PC/getpathp.c index b182ae6a58df68..1d18faed4500ed 100644 --- a/PC/getpathp.c +++ b/PC/getpathp.c @@ -624,7 +624,7 @@ read_pth_file(const wchar_t *path, wchar_t *prefix, int *isolated, int *nosite) static void -calculate_path(_PyCoreConfig *core_config) +calculate_path(_PyMainInterpreterConfig *config) { wchar_t argv0_path[MAXPATHLEN+1]; wchar_t *buf; @@ -637,8 +637,8 @@ calculate_path(_PyCoreConfig *core_config) wchar_t *userpath = NULL; wchar_t zip_path[MAXPATHLEN+1]; - if (core_config) { - envpath = core_config->module_search_path_env; + if (config) { + envpath = config->module_search_path_env; } else if (!Py_IgnoreEnvironmentFlag) { envpath = _wgetenv(L"PYTHONPATH"); @@ -899,10 +899,10 @@ Py_SetPath(const wchar_t *path) } wchar_t * -_Py_GetPathWithConfig(_PyCoreConfig *core_config) +_Py_GetPathWithConfig(_PyMainInterpreterConfig *config) { if (!module_search_path) { - calculate_path(core_config); + calculate_path(config); } return module_search_path; } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 9eeac9d33be601..552501d23ca531 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -843,7 +843,7 @@ _Py_InitializeMainInterpreter(const _PyMainInterpreterConfig *config) /* GetPath may initialize state that _PySys_EndInit locks in, and so has to be called first. */ /* TODO: Call Py_GetPath() in Py_ReadConfig, rather than here */ - wchar_t *sys_path = _Py_GetPathWithConfig(&interp->core_config); + wchar_t *sys_path = _Py_GetPathWithConfig(&interp->config); if (interp->core_config._disable_importlib) { /* Special mode for freeze_importlib: run with no import system @@ -1301,7 +1301,7 @@ new_interpreter(PyThreadState **tstate_p) /* XXX The following is lax in error checking */ - wchar_t *sys_path = _Py_GetPathWithConfig(&interp->core_config); + wchar_t *sys_path = _Py_GetPathWithConfig(&interp->config); PyObject *modules = PyDict_New(); if (modules == NULL) { From 1f15111a6e15d52f6b08907576ec61493cd59358 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 23 Nov 2017 10:43:14 +0100 Subject: [PATCH 11/75] bpo-32030: Add _PyMainInterpreterConfig.pythonhome (#4513) * Py_Main() now reads the PYTHONHOME environment variable * Add _Py_GetPythonHomeWithConfig() private function * Add _PyWarnings_InitWithConfig() * init_filters() doesn't get the current core configuration from the current interpreter or Python thread anymore. Pass explicitly the configuration to _PyWarnings_InitWithConfig(). * _Py_InitializeCore() now fails on _PyWarnings_InitWithConfig() failure. * Pass configuration as constant --- Include/pylifecycle.h | 7 +++- Include/pystate.h | 5 ++- Include/warnings.h | 3 ++ Modules/getpath.c | 6 ++-- Modules/main.c | 81 +++++++++++++++++++++++++++++++++++-------- PC/getpathp.c | 6 ++-- Python/_warnings.c | 20 +++++++---- Python/pylifecycle.c | 49 ++++++++++++++++++-------- 8 files changed, 135 insertions(+), 42 deletions(-) diff --git a/Include/pylifecycle.h b/Include/pylifecycle.h index 5eaa74edab8f98..3b603c87ad4db1 100644 --- a/Include/pylifecycle.h +++ b/Include/pylifecycle.h @@ -12,6 +12,10 @@ PyAPI_FUNC(wchar_t *) Py_GetProgramName(void); PyAPI_FUNC(void) Py_SetPythonHome(wchar_t *); PyAPI_FUNC(wchar_t *) Py_GetPythonHome(void); +#ifdef Py_BUILD_CORE +PyAPI_FUNC(wchar_t *) _Py_GetPythonHomeWithConfig( + const _PyMainInterpreterConfig *config); +#endif #ifndef Py_LIMITED_API /* Only used by applications that embed the interpreter and need to @@ -94,7 +98,8 @@ PyAPI_FUNC(wchar_t *) Py_GetPrefix(void); PyAPI_FUNC(wchar_t *) Py_GetExecPrefix(void); PyAPI_FUNC(wchar_t *) Py_GetPath(void); #ifdef Py_BUILD_CORE -PyAPI_FUNC(wchar_t *) _Py_GetPathWithConfig(_PyMainInterpreterConfig *config); +PyAPI_FUNC(wchar_t *) _Py_GetPathWithConfig( + const _PyMainInterpreterConfig *config); #endif PyAPI_FUNC(void) Py_SetPath(const wchar_t *); #ifdef MS_WINDOWS diff --git a/Include/pystate.h b/Include/pystate.h index b2739f1db25390..ab6400cddc8c55 100644 --- a/Include/pystate.h +++ b/Include/pystate.h @@ -61,12 +61,15 @@ typedef struct { typedef struct { int install_signal_handlers; wchar_t *module_search_path_env; /* PYTHONPATH environment variable */ + wchar_t *pythonhome; /* PYTHONHOME environment variable, + see also Py_SetPythonHome(). */ } _PyMainInterpreterConfig; #define _PyMainInterpreterConfig_INIT \ (_PyMainInterpreterConfig){\ .install_signal_handlers = -1, \ - .module_search_path_env = NULL} + .module_search_path_env = NULL, \ + .pythonhome = NULL} typedef struct _is { diff --git a/Include/warnings.h b/Include/warnings.h index a3f83ff6967e40..25f715e3a8bb05 100644 --- a/Include/warnings.h +++ b/Include/warnings.h @@ -7,6 +7,9 @@ extern "C" { #ifndef Py_LIMITED_API PyAPI_FUNC(PyObject*) _PyWarnings_Init(void); #endif +#ifdef Py_BUILD_CORE +PyAPI_FUNC(PyObject*) _PyWarnings_InitWithConfig(const _PyCoreConfig *config); +#endif PyAPI_FUNC(int) PyErr_WarnEx( PyObject *category, diff --git a/Modules/getpath.c b/Modules/getpath.c index ead143280b74e1..62f5e695849aa6 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -456,13 +456,13 @@ search_for_exec_prefix(wchar_t *argv0_path, wchar_t *home, } static void -calculate_path(_PyMainInterpreterConfig *config) +calculate_path(const _PyMainInterpreterConfig *config) { extern wchar_t *Py_GetProgramName(void); static const wchar_t delimiter[2] = {DELIM, '\0'}; static const wchar_t separator[2] = {SEP, '\0'}; - wchar_t *home = Py_GetPythonHome(); + wchar_t *home = _Py_GetPythonHomeWithConfig(config); char *_path = getenv("PATH"); wchar_t *path_buffer = NULL; wchar_t *path = NULL; @@ -858,7 +858,7 @@ Py_SetPath(const wchar_t *path) } wchar_t * -_Py_GetPathWithConfig(_PyMainInterpreterConfig *config) +_Py_GetPathWithConfig(const _PyMainInterpreterConfig *config) { if (!module_search_path) { calculate_path(config); diff --git a/Modules/main.c b/Modules/main.c index 8390af292383ce..07e0d2aa85ca49 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -400,7 +400,6 @@ typedef struct { _PyInitError err; /* PYTHONWARNINGS env var */ _Py_OptList env_warning_options; - /* PYTHONPATH env var */ int argc; wchar_t **argv; } _PyMain; @@ -1368,47 +1367,98 @@ pymain_set_flags_from_env(_PyMain *pymain) static int -pymain_init_pythonpath(_PyMain *pymain) +pymain_get_env_var_dup(_PyMain *pymain, wchar_t **dest, + wchar_t *wname, char *name) { if (Py_IgnoreEnvironmentFlag) { + *dest = NULL; return 0; } #ifdef MS_WINDOWS - wchar_t *path = _wgetenv(L"PYTHONPATH"); - if (!path || path[0] == '\0') { + wchar_t *var = _wgetenv(wname); + if (!var || var[0] == '\0') { + *dest = NULL; return 0; } - wchar_t *path2 = pymain_wstrdup(pymain, path); - if (path2 == NULL) { + wchar_t *copy = pymain_wstrdup(pymain, var); + if (copy == NULL) { return -1; } - pymain->config.module_search_path_env = path2; + *dest = copy; #else - char *path = pymain_get_env_var("PYTHONPATH"); - if (!path) { + char *var = getenv(name); + if (!var || var[0] == '\0') { + *dest = NULL; return 0; } size_t len; - wchar_t *wpath = Py_DecodeLocale(path, &len); - if (!wpath) { + wchar_t *wvar = Py_DecodeLocale(var, &len); + if (!wvar) { if (len == (size_t)-2) { - pymain->err = _Py_INIT_ERR("failed to decode PYTHONHOME"); + /* don't set pymain->err */ + return -2; } else { pymain->err = INIT_NO_MEMORY(); + return -1; } - return -1; } - pymain->config.module_search_path_env = wpath; + *dest = wvar; #endif return 0; } +static int +pymain_init_pythonpath(_PyMain *pymain) +{ + wchar_t *path; + int res = pymain_get_env_var_dup(pymain, &path, + L"PYTHONPATH", "PYTHONPATH"); + if (res < 0) { + if (res == -2) { + pymain->err = _Py_INIT_ERR("failed to decode PYTHONPATH"); + } + return -1; + } + pymain->config.module_search_path_env = path; + return 0; +} + + +static int +pymain_init_pythonhome(_PyMain *pymain) +{ + wchar_t *home; + + home = Py_GetPythonHome(); + if (home) { + /* Py_SetPythonHome() has been called before Py_Main(), + use its value */ + pymain->config.pythonhome = pymain_wstrdup(pymain, home); + if (pymain->config.pythonhome == NULL) { + return -1; + } + return 0; + } + + int res = pymain_get_env_var_dup(pymain, &home, + L"PYTHONHOME", "PYTHONHOME"); + if (res < 0) { + if (res == -2) { + pymain->err = _Py_INIT_ERR("failed to decode PYTHONHOME"); + } + return -1; + } + pymain->config.pythonhome = home; + return 0; +} + + static int pymain_parse_envvars(_PyMain *pymain) { @@ -1433,6 +1483,9 @@ pymain_parse_envvars(_PyMain *pymain) if (pymain_init_pythonpath(pymain) < 0) { return -1; } + if (pymain_init_pythonhome(pymain) < 0) { + return -1; + } /* -X options */ if (pymain_get_xoption(pymain, L"showrefcount")) { diff --git a/PC/getpathp.c b/PC/getpathp.c index 1d18faed4500ed..4756dc8abbb620 100644 --- a/PC/getpathp.c +++ b/PC/getpathp.c @@ -624,12 +624,12 @@ read_pth_file(const wchar_t *path, wchar_t *prefix, int *isolated, int *nosite) static void -calculate_path(_PyMainInterpreterConfig *config) +calculate_path(const _PyMainInterpreterConfig *config) { wchar_t argv0_path[MAXPATHLEN+1]; wchar_t *buf; size_t bufsz; - wchar_t *pythonhome = Py_GetPythonHome(); + wchar_t *pythonhome = _Py_GetPythonHomeWithConfig(config); wchar_t *envpath = NULL; int skiphome, skipdefault; @@ -899,7 +899,7 @@ Py_SetPath(const wchar_t *path) } wchar_t * -_Py_GetPathWithConfig(_PyMainInterpreterConfig *config) +_Py_GetPathWithConfig(const _PyMainInterpreterConfig *config) { if (!module_search_path) { calculate_path(config); diff --git a/Python/_warnings.c b/Python/_warnings.c index d865f0ad2c089a..36d649fda10c54 100644 --- a/Python/_warnings.c +++ b/Python/_warnings.c @@ -1185,10 +1185,9 @@ create_filter(PyObject *category, const char *action) } static PyObject * -init_filters(void) +init_filters(const _PyCoreConfig *config) { - PyInterpreterState *interp = PyThreadState_GET()->interp; - int dev_mode = interp->core_config.dev_mode; + int dev_mode = config->dev_mode; Py_ssize_t count = 2; if (dev_mode) { @@ -1264,8 +1263,8 @@ static struct PyModuleDef warningsmodule = { }; -PyMODINIT_FUNC -_PyWarnings_Init(void) +PyObject* +_PyWarnings_InitWithConfig(const _PyCoreConfig *config) { PyObject *m; @@ -1274,7 +1273,7 @@ _PyWarnings_Init(void) return NULL; if (_PyRuntime.warnings.filters == NULL) { - _PyRuntime.warnings.filters = init_filters(); + _PyRuntime.warnings.filters = init_filters(config); if (_PyRuntime.warnings.filters == NULL) return NULL; } @@ -1305,3 +1304,12 @@ _PyWarnings_Init(void) _PyRuntime.warnings.filters_version = 0; return m; } + + +PyMODINIT_FUNC +_PyWarnings_Init(void) +{ + PyInterpreterState *interp = PyThreadState_GET()->interp; + const _PyCoreConfig *config = &interp->core_config; + return _PyWarnings_InitWithConfig(config); +} diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 552501d23ca531..5bbbbc68f08ecc 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -767,7 +767,9 @@ _Py_InitializeCore(const _PyCoreConfig *config) } /* Initialize _warnings. */ - _PyWarnings_Init(); + if (_PyWarnings_InitWithConfig(&interp->core_config) == NULL) { + return _Py_INIT_ERR("can't initialize warnings"); + } /* This call sets up builtin and frozen import support */ if (!interp->core_config._disable_importlib) { @@ -880,7 +882,7 @@ _Py_InitializeMainInterpreter(const _PyMainInterpreterConfig *config) return err; } - if (config->install_signal_handlers) { + if (interp->config.install_signal_handlers) { err = initsigs(); /* Signal handling stuff, including initintr() */ if (_Py_INIT_FAILED(err)) { return err; @@ -1468,7 +1470,6 @@ Py_GetProgramName(void) } static wchar_t *default_home = NULL; -static wchar_t env_home[MAXPATHLEN+1]; void Py_SetPythonHome(wchar_t *home) @@ -1477,20 +1478,40 @@ Py_SetPythonHome(wchar_t *home) } wchar_t * -Py_GetPythonHome(void) +_Py_GetPythonHomeWithConfig(const _PyMainInterpreterConfig *config) { - wchar_t *home = default_home; - if (home == NULL && !Py_IgnoreEnvironmentFlag) { - char* chome = Py_GETENV("PYTHONHOME"); - if (chome) { - size_t size = Py_ARRAY_LENGTH(env_home); - size_t r = mbstowcs(env_home, chome, size); - if (r != (size_t)-1 && r < size) - home = env_home; - } + /* Use a static buffer to avoid heap memory allocation failure. + Py_GetPythonHome() doesn't allow to report error, and the caller + doesn't release memory. */ + static wchar_t buffer[MAXPATHLEN+1]; + + if (default_home) { + return default_home; + } + if (config) { + return config->pythonhome; } - return home; + + char *home = Py_GETENV("PYTHONHOME"); + if (!home) { + return NULL; + } + + size_t size = Py_ARRAY_LENGTH(buffer); + size_t r = mbstowcs(buffer, home, size); + if (r == (size_t)-1 || r >= size) { + /* conversion failed or the static buffer is too small */ + return NULL; + } + + return buffer; +} + +wchar_t * +Py_GetPythonHome(void) +{ + return _Py_GetPythonHomeWithConfig(NULL); } /* Add the __main__ module */ From 5ad7ef8e420de8a54fb30ed37362194c6b96012c Mon Sep 17 00:00:00 2001 From: xdegaye Date: Thu, 23 Nov 2017 11:13:22 +0100 Subject: [PATCH 12/75] bpo-28538: Revert all the changes (now using Android Unified Headers) (GH-4479) --- .../2017-11-21-17-27-59.bpo-28538.DsNBS7.rst | 2 + configure | 60 +------------------ configure.ac | 26 +------- 3 files changed, 5 insertions(+), 83 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2017-11-21-17-27-59.bpo-28538.DsNBS7.rst diff --git a/Misc/NEWS.d/next/Build/2017-11-21-17-27-59.bpo-28538.DsNBS7.rst b/Misc/NEWS.d/next/Build/2017-11-21-17-27-59.bpo-28538.DsNBS7.rst new file mode 100644 index 00000000000000..db435b008f724c --- /dev/null +++ b/Misc/NEWS.d/next/Build/2017-11-21-17-27-59.bpo-28538.DsNBS7.rst @@ -0,0 +1,2 @@ +Revert the previous changes, the if_nameindex structure is defined by +Unified Headers. diff --git a/configure b/configure index 5e0522476e7126..ab33115d56398d 100755 --- a/configure +++ b/configure @@ -778,7 +778,6 @@ infodir docdir oldincludedir includedir -runstatedir localstatedir sharedstatedir sysconfdir @@ -890,7 +889,6 @@ datadir='${datarootdir}' sysconfdir='${prefix}/etc' sharedstatedir='${prefix}/com' localstatedir='${prefix}/var' -runstatedir='${localstatedir}/run' includedir='${prefix}/include' oldincludedir='/usr/include' docdir='${datarootdir}/doc/${PACKAGE_TARNAME}' @@ -1143,15 +1141,6 @@ do | -silent | --silent | --silen | --sile | --sil) silent=yes ;; - -runstatedir | --runstatedir | --runstatedi | --runstated \ - | --runstate | --runstat | --runsta | --runst | --runs \ - | --run | --ru | --r) - ac_prev=runstatedir ;; - -runstatedir=* | --runstatedir=* | --runstatedi=* | --runstated=* \ - | --runstate=* | --runstat=* | --runsta=* | --runst=* | --runs=* \ - | --run=* | --ru=* | --r=*) - runstatedir=$ac_optarg ;; - -sbindir | --sbindir | --sbindi | --sbind | --sbin | --sbi | --sb) ac_prev=sbindir ;; -sbindir=* | --sbindir=* | --sbindi=* | --sbind=* | --sbin=* \ @@ -1289,7 +1278,7 @@ fi for ac_var in exec_prefix prefix bindir sbindir libexecdir datarootdir \ datadir sysconfdir sharedstatedir localstatedir includedir \ oldincludedir docdir infodir htmldir dvidir pdfdir psdir \ - libdir localedir mandir runstatedir + libdir localedir mandir do eval ac_val=\$$ac_var # Remove trailing slashes. @@ -1442,7 +1431,6 @@ Fine tuning of the installation directories: --sysconfdir=DIR read-only single-machine data [PREFIX/etc] --sharedstatedir=DIR modifiable architecture-independent data [PREFIX/com] --localstatedir=DIR modifiable single-machine data [PREFIX/var] - --runstatedir=DIR modifiable per-process data [LOCALSTATEDIR/run] --libdir=DIR object code libraries [EPREFIX/lib] --includedir=DIR C header files [PREFIX/include] --oldincludedir=DIR C header files for non-gcc [/usr/include] @@ -11155,6 +11143,7 @@ for ac_func in alarm accept4 setitimer getitimer bind_textdomain_codeset chown \ futimens futimes gai_strerror getentropy \ getgrouplist getgroups getlogin getloadavg getpeername getpgid getpid \ getpriority getresuid getresgid getpwent getspnam getspent getsid getwd \ + if_nameindex \ initgroups kill killpg lchmod lchown linkat lstat lutimes mmap \ memrchr mbrtowc mkdirat mkfifo \ mkfifoat mknod mknodat mktime mremap nice openat pathconf pause pipe2 plock poll \ @@ -12585,51 +12574,6 @@ else { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 $as_echo "no" >&6; } -fi -rm -f core conftest.err conftest.$ac_objext \ - conftest$ac_exeext conftest.$ac_ext - -# On Android API level 24 with android-ndk-r13, if_nameindex() is available, -# but the if_nameindex structure is not defined. -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for if_nameindex" >&5 -$as_echo_n "checking for if_nameindex... " >&6; } -cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ - -#include -#ifdef STDC_HEADERS -# include -# include -#else -# ifdef HAVE_STDLIB_H -# include -# endif -#endif -#ifdef HAVE_SYS_SOCKET_H -# include -#endif -#ifdef HAVE_NET_IF_H -# include -#endif - -int -main () -{ -struct if_nameindex *ni = if_nameindex(); int x = ni[0].if_index; - ; - return 0; -} -_ACEOF -if ac_fn_c_try_link "$LINENO"; then : - -$as_echo "#define HAVE_IF_NAMEINDEX 1" >>confdefs.h - - { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 -$as_echo "yes" >&6; } -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } - fi rm -f core conftest.err conftest.$ac_objext \ conftest$ac_exeext conftest.$ac_ext diff --git a/configure.ac b/configure.ac index 3464212eddc019..501f07892f8575 100644 --- a/configure.ac +++ b/configure.ac @@ -3411,6 +3411,7 @@ AC_CHECK_FUNCS(alarm accept4 setitimer getitimer bind_textdomain_codeset chown \ futimens futimes gai_strerror getentropy \ getgrouplist getgroups getlogin getloadavg getpeername getpgid getpid \ getpriority getresuid getresgid getpwent getspnam getspent getsid getwd \ + if_nameindex \ initgroups kill killpg lchmod lchown linkat lstat lutimes mmap \ memrchr mbrtowc mkdirat mkfifo \ mkfifoat mknod mknodat mktime mremap nice openat pathconf pause pipe2 plock poll \ @@ -3760,31 +3761,6 @@ AC_LINK_IFELSE([AC_LANG_PROGRAM([[ AC_MSG_RESULT(no) ]) -# On Android API level 24 with android-ndk-r13, if_nameindex() is available, -# but the if_nameindex structure is not defined. -AC_MSG_CHECKING(for if_nameindex) -AC_LINK_IFELSE([AC_LANG_PROGRAM([[ -#include -#ifdef STDC_HEADERS -# include -# include -#else -# ifdef HAVE_STDLIB_H -# include -# endif -#endif -#ifdef HAVE_SYS_SOCKET_H -# include -#endif -#ifdef HAVE_NET_IF_H -# include -#endif -]], [[struct if_nameindex *ni = if_nameindex(); int x = ni[0].if_index;]])], - [AC_DEFINE(HAVE_IF_NAMEINDEX, 1, Define to 1 if you have the 'if_nameindex' function.) - AC_MSG_RESULT(yes)], - [AC_MSG_RESULT(no) -]) - # Issue #28762: lockf() is available on Android API level 24, but the F_LOCK # macro is not defined in android-ndk-r13. AC_MSG_CHECKING(for lockf) From c06c22e9a9fb9326e79fcf1551601eacc1fd457d Mon Sep 17 00:00:00 2001 From: xdegaye Date: Thu, 23 Nov 2017 11:44:38 +0100 Subject: [PATCH 13/75] bpo-29040: Support building Android with Unified Headers (GH-4492) --- Include/pyport.h | 4 +++- .../next/Build/2017-11-21-16-56-24.bpo-29040.14lCSr.rst | 2 ++ configure | 5 ++++- configure.ac | 5 ++++- 4 files changed, 13 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2017-11-21-16-56-24.bpo-29040.14lCSr.rst diff --git a/Include/pyport.h b/Include/pyport.h index 0e82543ac78355..f2e247a374e063 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -784,7 +784,9 @@ extern _invalid_parameter_handler _Py_silent_invalid_parameter_handler; #endif /* Py_BUILD_CORE */ #ifdef __ANDROID__ -#include +/* The Android langinfo.h header is not used. */ +#undef HAVE_LANGINFO_H +#undef CODESET #endif /* Maximum value of the Windows DWORD type */ diff --git a/Misc/NEWS.d/next/Build/2017-11-21-16-56-24.bpo-29040.14lCSr.rst b/Misc/NEWS.d/next/Build/2017-11-21-16-56-24.bpo-29040.14lCSr.rst new file mode 100644 index 00000000000000..60f05db8d4f043 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2017-11-21-16-56-24.bpo-29040.14lCSr.rst @@ -0,0 +1,2 @@ +Support building Android with Unified Headers. The first NDK release to +support Unified Headers is android-ndk-r14. diff --git a/configure b/configure index ab33115d56398d..969e1d51c6efe6 100755 --- a/configure +++ b/configure @@ -5611,7 +5611,6 @@ $as_echo "$ac_cv_safe_to_define___extensions__" >&6; } $as_echo_n "checking for the Android API level... " >&6; } cat >> conftest.c < android_api = __ANDROID_API__ arm_arch = __ARM_ARCH #else @@ -5624,6 +5623,10 @@ if $CPP $CPPFLAGS conftest.c >conftest.out 2>/dev/null; then _arm_arch=`sed -n -e '/__ARM_ARCH/d' -e 's/^arm_arch = //p' conftest.out` { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ANDROID_API_LEVEL" >&5 $as_echo "$ANDROID_API_LEVEL" >&6; } + if test -z "$ANDROID_API_LEVEL"; then + echo 'Fatal: you must define __ANDROID_API__' + exit 1 + fi cat >>confdefs.h <<_ACEOF #define ANDROID_API_LEVEL $ANDROID_API_LEVEL diff --git a/configure.ac b/configure.ac index 501f07892f8575..6a69f789fe1774 100644 --- a/configure.ac +++ b/configure.ac @@ -885,7 +885,6 @@ AC_USE_SYSTEM_EXTENSIONS AC_MSG_CHECKING([for the Android API level]) cat >> conftest.c < android_api = __ANDROID_API__ arm_arch = __ARM_ARCH #else @@ -897,6 +896,10 @@ if $CPP $CPPFLAGS conftest.c >conftest.out 2>/dev/null; then ANDROID_API_LEVEL=`sed -n -e '/__ANDROID_API__/d' -e 's/^android_api = //p' conftest.out` _arm_arch=`sed -n -e '/__ARM_ARCH/d' -e 's/^arm_arch = //p' conftest.out` AC_MSG_RESULT([$ANDROID_API_LEVEL]) + if test -z "$ANDROID_API_LEVEL"; then + echo 'Fatal: you must define __ANDROID_API__' + exit 1 + fi AC_DEFINE_UNQUOTED(ANDROID_API_LEVEL, $ANDROID_API_LEVEL, [The Android API level.]) AC_MSG_CHECKING([for the Android arm ABI]) From 5ce1069a6ff0d5443074d33ba1d403ccd2eaf3d3 Mon Sep 17 00:00:00 2001 From: xdegaye Date: Thu, 23 Nov 2017 12:01:36 +0100 Subject: [PATCH 14/75] bpo-28762: Revert last commit (now using Android Unified Headers) (GH-4488) --- .../2017-11-21-17-12-24.bpo-28762.R6uu8w.rst | 1 + configure | 31 +------------------ configure.ac | 11 +------ pyconfig.h.in | 2 +- 4 files changed, 4 insertions(+), 41 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2017-11-21-17-12-24.bpo-28762.R6uu8w.rst diff --git a/Misc/NEWS.d/next/Build/2017-11-21-17-12-24.bpo-28762.R6uu8w.rst b/Misc/NEWS.d/next/Build/2017-11-21-17-12-24.bpo-28762.R6uu8w.rst new file mode 100644 index 00000000000000..0c57e33c0a5221 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2017-11-21-17-12-24.bpo-28762.R6uu8w.rst @@ -0,0 +1 @@ +Revert the last commit, the F_LOCK macro is defined by Android Unified Headers. diff --git a/configure b/configure index 969e1d51c6efe6..d02675742d27d9 100755 --- a/configure +++ b/configure @@ -11147,7 +11147,7 @@ for ac_func in alarm accept4 setitimer getitimer bind_textdomain_codeset chown \ getgrouplist getgroups getlogin getloadavg getpeername getpgid getpid \ getpriority getresuid getresgid getpwent getspnam getspent getsid getwd \ if_nameindex \ - initgroups kill killpg lchmod lchown linkat lstat lutimes mmap \ + initgroups kill killpg lchmod lchown lockf linkat lstat lutimes mmap \ memrchr mbrtowc mkdirat mkfifo \ mkfifoat mknod mknodat mktime mremap nice openat pathconf pause pipe2 plock poll \ posix_fallocate posix_fadvise pread \ @@ -12577,35 +12577,6 @@ else { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 $as_echo "no" >&6; } -fi -rm -f core conftest.err conftest.$ac_objext \ - conftest$ac_exeext conftest.$ac_ext - -# Issue #28762: lockf() is available on Android API level 24, but the F_LOCK -# macro is not defined in android-ndk-r13. -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for lockf" >&5 -$as_echo_n "checking for lockf... " >&6; } -cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ -#include -int -main () -{ -lockf(0, F_LOCK, 0); - ; - return 0; -} -_ACEOF -if ac_fn_c_try_link "$LINENO"; then : - -$as_echo "#define HAVE_LOCKF 1" >>confdefs.h - - { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 -$as_echo "yes" >&6; } -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } - fi rm -f core conftest.err conftest.$ac_objext \ conftest$ac_exeext conftest.$ac_ext diff --git a/configure.ac b/configure.ac index 6a69f789fe1774..68a95c3be6af4a 100644 --- a/configure.ac +++ b/configure.ac @@ -3415,7 +3415,7 @@ AC_CHECK_FUNCS(alarm accept4 setitimer getitimer bind_textdomain_codeset chown \ getgrouplist getgroups getlogin getloadavg getpeername getpgid getpid \ getpriority getresuid getresgid getpwent getspnam getspent getsid getwd \ if_nameindex \ - initgroups kill killpg lchmod lchown linkat lstat lutimes mmap \ + initgroups kill killpg lchmod lchown lockf linkat lstat lutimes mmap \ memrchr mbrtowc mkdirat mkfifo \ mkfifoat mknod mknodat mktime mremap nice openat pathconf pause pipe2 plock poll \ posix_fallocate posix_fadvise pread \ @@ -3764,15 +3764,6 @@ AC_LINK_IFELSE([AC_LANG_PROGRAM([[ AC_MSG_RESULT(no) ]) -# Issue #28762: lockf() is available on Android API level 24, but the F_LOCK -# macro is not defined in android-ndk-r13. -AC_MSG_CHECKING(for lockf) -AC_LINK_IFELSE([AC_LANG_PROGRAM([[#include ]],[[lockf(0, F_LOCK, 0);]])], - [AC_DEFINE(HAVE_LOCKF, 1, Define to 1 if you have the 'lockf' function and the F_LOCK macro.) - AC_MSG_RESULT(yes)], - [AC_MSG_RESULT(no) -]) - # On OSF/1 V5.1, getaddrinfo is available, but a define # for [no]getaddrinfo in netdb.h. AC_MSG_CHECKING(for getaddrinfo) diff --git a/pyconfig.h.in b/pyconfig.h.in index 6e0f3e8eeb37b5..66b9e888274500 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -601,7 +601,7 @@ /* Define to 1 if you have the header file. */ #undef HAVE_LINUX_VM_SOCKETS_H -/* Define to 1 if you have the 'lockf' function and the F_LOCK macro. */ +/* Define to 1 if you have the `lockf' function. */ #undef HAVE_LOCKF /* Define to 1 if you have the `log1p' function. */ From bdb8315c21825487b54852ff0511fb4881ea2181 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Thu, 23 Nov 2017 15:47:30 +0300 Subject: [PATCH 15/75] bpo-1102: View.Fetch() now returns None when it's exhausted (GH-4459) --- Lib/test/test_msilib.py | 41 ++++++++++++++++++- .../2017-11-19-09-46-27.bpo-1102.NY-g1F.rst | 4 ++ PC/_msi.c | 6 ++- 3 files changed, 49 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Windows/2017-11-19-09-46-27.bpo-1102.NY-g1F.rst diff --git a/Lib/test/test_msilib.py b/Lib/test/test_msilib.py index f656f7234e2d33..65ff3869c33cbe 100644 --- a/Lib/test/test_msilib.py +++ b/Lib/test/test_msilib.py @@ -1,7 +1,46 @@ """ Test suite for the code in msilib """ import unittest -from test.support import import_module +from test.support import TESTFN, import_module, unlink msilib = import_module('msilib') +import msilib.schema + + +def init_database(): + path = TESTFN + '.msi' + db = msilib.init_database( + path, + msilib.schema, + 'Python Tests', + 'product_code', + '1.0', + 'PSF', + ) + return db, path + + +class MsiDatabaseTestCase(unittest.TestCase): + + def test_view_fetch_returns_none(self): + db, db_path = init_database() + properties = [] + view = db.OpenView('SELECT Property, Value FROM Property') + view.Execute(None) + while True: + record = view.Fetch() + if record is None: + break + properties.append(record.GetString(1)) + view.Close() + db.Close() + self.assertEqual( + properties, + [ + 'ProductName', 'ProductCode', 'ProductVersion', + 'Manufacturer', 'ProductLanguage', + ] + ) + self.addCleanup(unlink, db_path) + class Test_make_id(unittest.TestCase): #http://msdn.microsoft.com/en-us/library/aa369212(v=vs.85).aspx diff --git a/Misc/NEWS.d/next/Windows/2017-11-19-09-46-27.bpo-1102.NY-g1F.rst b/Misc/NEWS.d/next/Windows/2017-11-19-09-46-27.bpo-1102.NY-g1F.rst new file mode 100644 index 00000000000000..6a6618e9692558 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2017-11-19-09-46-27.bpo-1102.NY-g1F.rst @@ -0,0 +1,4 @@ +Return ``None`` when ``View.Fetch()`` returns ``ERROR_NO_MORE_ITEMS`` +instead of raising ``MSIError``. + +Initial patch by Anthony Tuininga. diff --git a/PC/_msi.c b/PC/_msi.c index df6c881b4ec44f..a6a12e2010738b 100644 --- a/PC/_msi.c +++ b/PC/_msi.c @@ -723,8 +723,12 @@ view_fetch(msiobj *view, PyObject*args) int status; MSIHANDLE result; - if ((status = MsiViewFetch(view->h, &result)) != ERROR_SUCCESS) + status = MsiViewFetch(view->h, &result); + if (status == ERROR_NO_MORE_ITEMS) { + Py_RETURN_NONE; + } else if (status != ERROR_SUCCESS) { return msierror(status); + } return record_new(result); } From 24c4db29f03284821df326f71db35f190f49eb59 Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Thu, 23 Nov 2017 10:16:21 -0500 Subject: [PATCH 16/75] Several sleep-improved fixes * Add a helper function to determine whether a MAC address is universally administered or not. * In the various MAC getter functions, keep track of the first locally administered MAC, and if no universally administered MAC is found, return the first local MAC. * In the ultimate fallback _random_getnode(), add a comment for the multicast bit being set, and use a better bitmask * In getnode(), just fall back to _random_getnode() explicitly if no other MAC has been found. --- Lib/test/test_uuid.py | 2 +- Lib/uuid.py | 45 +++++++++++++++++++++++++++++++------------ 2 files changed, 34 insertions(+), 13 deletions(-) diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index 19564f2d548d28..97f34cdbe3310f 100644 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -562,7 +562,7 @@ def test_netbios_getnode(self): def test_random_getnode(self): node = self.uuid._random_getnode() # Least significant bit of first octet must be set. - self.assertTrue(node & 0x010000000000, '%012x' % node) + self.assertTrue(node & (1 << 40), '%012x' % node) self.check_node(node) @unittest.skipUnless(os.name == 'posix', 'requires Posix') diff --git a/Lib/uuid.py b/Lib/uuid.py index 0535433f2bffda..3eec69f6f99d4b 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -354,10 +354,13 @@ def _popen(command, *args): # significant, or 1<<41. We'll skip over any locally administered MAC # addresses, as it makes no sense to use those in UUID calculation. # -# For a good, simple explanation, see the section on Universal vs. local in -# this page: https://en.wikipedia.org/wiki/MAC_address +# See https://en.wikipedia.org/wiki/MAC_address#Universal_vs._local + +def _is_universal(mac): + return not (mac & (1 << 41)) def _find_mac(command, args, hw_identifiers, get_index): + first_local_mac = None try: proc = _popen(command, *args.split()) if not proc: @@ -370,8 +373,9 @@ def _find_mac(command, args, hw_identifiers, get_index): try: word = words[get_index(i)] mac = int(word.replace(b':', b''), 16) - if ~(mac & (1<<41)): + if _is_universal(mac): return mac + first_local_mac = first_local_mac or mac except (ValueError, IndexError): # Virtual interfaces, such as those provided by # VPNs, do not have a colon-delimited MAC address @@ -381,7 +385,7 @@ def _find_mac(command, args, hw_identifiers, get_index): pass except OSError: pass - return None + return first_local_mac or None def _ifconfig_getnode(): """Get the hardware address on Unix by running ifconfig.""" @@ -435,6 +439,7 @@ def _lanscan_getnode(): def _netstat_getnode(): """Get the hardware address on Unix by running netstat.""" # This might work on AIX, Tru64 UNIX. + first_local_mac = None try: proc = _popen('netstat', '-ia') if not proc: @@ -451,17 +456,19 @@ def _netstat_getnode(): word = words[i] if len(word) == 17 and word.count(b':') == 5: mac = int(word.replace(b':', b''), 16) - if ~(mac & (1<<41)): + if _is_universal(mac): return mac + first_local_mac = first_local_mac or mac except (ValueError, IndexError): pass except OSError: pass - return None + return first_local_mac or None def _ipconfig_getnode(): """Get the hardware address on Windows by running ipconfig.exe.""" import os, re + first_local_mac = None dirs = ['', r'c:\windows\system32', r'c:\winnt\system32'] try: import ctypes @@ -480,14 +487,16 @@ def _ipconfig_getnode(): value = line.split(':')[-1].strip().lower() if re.match('([0-9a-f][0-9a-f]-){5}[0-9a-f][0-9a-f]', value): mac = int(value.replace('-', ''), 16) - if ~(mac & (1<<41)): + if _is_universal(mac): return mac - return None + first_local_mac = first_local_mac or mac + return first_local_mac or None def _netbios_getnode(): """Get the hardware address on Windows using NetBIOS calls. See http://support.microsoft.com/kb/118623 for details.""" import win32wnet, netbios + first_local_mac = None ncb = netbios.NCB() ncb.Command = netbios.NCBENUM ncb.Buffer = adapters = netbios.LANA_ENUM() @@ -513,8 +522,9 @@ def _netbios_getnode(): if len(bytes) != 6: continue mac = int.from_bytes(bytes, 'big') - if ~(mac & (1<<41)): + if _is_universal(mac): return mac + first_local_mac = first_local_mac or mac return None @@ -628,9 +638,19 @@ def _windll_getnode(): return UUID(bytes=bytes_(_buffer.raw)).node def _random_getnode(): - """Get a random node ID, with eighth bit set as suggested by RFC 4122.""" + """Get a random node ID.""" + # RFC 4122, $4.1.6. says "For systems with no IEEE address, a randomly or + # pseudo-randomly generated value may be used; see Section 4.5. The + # multicast bit must be set in such addresses, in order that they will + # never conflict with addresses obtained from network cards." + # + # The "multicast bit" of a MAC address is defined to be "the least + # significant bit of the first octet". This worlds out to be the 41st bit + # counting from 1 being the least significant bit, or 1<<40. + # + # See https://en.wikipedia.org/wiki/MAC_address#Unicast_vs._multicast import random - return random.getrandbits(48) | 0x010000000000 + return random.getrandbits(48) | (1 << 40) _node = None @@ -653,13 +673,14 @@ def getnode(): getters = [_unix_getnode, _ifconfig_getnode, _ip_getnode, _arp_getnode, _lanscan_getnode, _netstat_getnode] - for getter in getters + [_random_getnode]: + for getter in getters: try: _node = getter() except: continue if _node is not None: return _node + return _random_getnode() _last_timestamp = None From c056584d287cbd1cac1808fc1ca92e0704930bdf Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Thu, 23 Nov 2017 10:25:51 -0500 Subject: [PATCH 17/75] A missed return --- Lib/uuid.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/uuid.py b/Lib/uuid.py index 3eec69f6f99d4b..364b5b0272b785 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -525,7 +525,7 @@ def _netbios_getnode(): if _is_universal(mac): return mac first_local_mac = first_local_mac or mac - return None + return first_local_mac or None _generate_time_safe = _UuidCreate = None From 7fd446e0d84ee8616b9449f0a1e924581d0fef03 Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Thu, 23 Nov 2017 10:27:21 -0500 Subject: [PATCH 18/75] Two typos --- Lib/uuid.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/uuid.py b/Lib/uuid.py index 364b5b0272b785..9e7c672f7a0a6f 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -639,13 +639,13 @@ def _windll_getnode(): def _random_getnode(): """Get a random node ID.""" - # RFC 4122, $4.1.6. says "For systems with no IEEE address, a randomly or + # RFC 4122, $4.1.6 says "For systems with no IEEE address, a randomly or # pseudo-randomly generated value may be used; see Section 4.5. The # multicast bit must be set in such addresses, in order that they will # never conflict with addresses obtained from network cards." # # The "multicast bit" of a MAC address is defined to be "the least - # significant bit of the first octet". This worlds out to be the 41st bit + # significant bit of the first octet". This works out to be the 41st bit # counting from 1 being the least significant bit, or 1<<40. # # See https://en.wikipedia.org/wiki/MAC_address#Unicast_vs._multicast From 0327bde9da203bb256b58218d012ca76ad0db4e4 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 23 Nov 2017 17:03:20 +0100 Subject: [PATCH 19/75] bpo-32030: Rewrite calculate_path() (#4521) * calculate_path() rewritten in Modules/getpath.c and PC/getpathp.c * Move global variables into a new PyPathConfig structure. * calculate_path(): * Split the huge calculate_path() function into subfunctions. * Add PyCalculatePath structure to pass data between subfunctions. * Document PyCalculatePath fields. * Move cleanup code into a new calculate_free() subfunction * calculate_init() now handles Py_DecodeLocale() failures properly * calculate_path() is now atomic: only replace PyPathConfig (path_config) at once on success. * _Py_GetPythonHomeWithConfig() now returns an error on failure * Add _Py_INIT_NO_MEMORY() helper: report a memory allocation failure * Coding style fixes (PEP 7) --- Include/pylifecycle.h | 38 +- Modules/getpath.c | 855 +++++++++++++++++++++++++----------------- Modules/main.c | 56 ++- PC/getpathp.c | 593 +++++++++++++++++++---------- Python/pylifecycle.c | 25 +- 5 files changed, 966 insertions(+), 601 deletions(-) diff --git a/Include/pylifecycle.h b/Include/pylifecycle.h index 3b603c87ad4db1..efdc58eed8ebf2 100644 --- a/Include/pylifecycle.h +++ b/Include/pylifecycle.h @@ -7,23 +7,7 @@ extern "C" { #endif -PyAPI_FUNC(void) Py_SetProgramName(wchar_t *); -PyAPI_FUNC(wchar_t *) Py_GetProgramName(void); - -PyAPI_FUNC(void) Py_SetPythonHome(wchar_t *); -PyAPI_FUNC(wchar_t *) Py_GetPythonHome(void); -#ifdef Py_BUILD_CORE -PyAPI_FUNC(wchar_t *) _Py_GetPythonHomeWithConfig( - const _PyMainInterpreterConfig *config); -#endif - #ifndef Py_LIMITED_API -/* Only used by applications that embed the interpreter and need to - * override the standard encoding determination mechanism - */ -PyAPI_FUNC(int) Py_SetStandardStreamEncoding(const char *encoding, - const char *errors); - typedef struct { const char *prefix; const char *msg; @@ -46,9 +30,31 @@ typedef struct { Don't abort() the process on such error. */ #define _Py_INIT_USER_ERR(MSG) \ (_PyInitError){.prefix = _Py_INIT_GET_FUNC(), .msg = (MSG), .user_err = 1} +#define _Py_INIT_NO_MEMORY() _Py_INIT_ERR("memory allocation failed") #define _Py_INIT_FAILED(err) \ (err.msg != NULL) +#endif + + +PyAPI_FUNC(void) Py_SetProgramName(wchar_t *); +PyAPI_FUNC(wchar_t *) Py_GetProgramName(void); + +PyAPI_FUNC(void) Py_SetPythonHome(wchar_t *); +PyAPI_FUNC(wchar_t *) Py_GetPythonHome(void); +#ifdef Py_BUILD_CORE +PyAPI_FUNC(_PyInitError) _Py_GetPythonHomeWithConfig( + const _PyMainInterpreterConfig *config, + wchar_t **home); +#endif + +#ifndef Py_LIMITED_API +/* Only used by applications that embed the interpreter and need to + * override the standard encoding determination mechanism + */ +PyAPI_FUNC(int) Py_SetStandardStreamEncoding(const char *encoding, + const char *errors); + /* PEP 432 Multi-phase initialization API (Private while provisional!) */ PyAPI_FUNC(_PyInitError) _Py_InitializeCore(const _PyCoreConfig *); PyAPI_FUNC(int) _Py_IsCoreInitialized(void); diff --git a/Modules/getpath.c b/Modules/getpath.c index 62f5e695849aa6..d8125ae2cac8eb 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -7,7 +7,7 @@ #include #ifdef __APPLE__ -#include +# include #endif /* Search in some common locations for the associated Python libraries. @@ -97,7 +97,7 @@ */ #ifdef __cplusplus - extern "C" { +extern "C" { #endif @@ -109,13 +109,38 @@ #define LANDMARK L"os.py" #endif -static wchar_t prefix[MAXPATHLEN+1]; -static wchar_t exec_prefix[MAXPATHLEN+1]; -static wchar_t progpath[MAXPATHLEN+1]; -static wchar_t *module_search_path = NULL; +typedef struct { + wchar_t prefix[MAXPATHLEN+1]; + wchar_t exec_prefix[MAXPATHLEN+1]; + wchar_t progpath[MAXPATHLEN+1]; + wchar_t *module_search_path; +} PyPathConfig; + +typedef struct { + wchar_t *path_env; /* PATH environment variable */ + wchar_t *home; /* PYTHONHOME environment variable */ + wchar_t *module_search_path_env; /* PYTHONPATH environment variable */ + wchar_t *module_search_path_buffer; + + wchar_t *prog; /* Program name */ + wchar_t *pythonpath; /* PYTHONPATH define */ + wchar_t *prefix; /* PREFIX define */ + wchar_t *exec_prefix; /* EXEC_PREFIX define */ + + wchar_t *lib_python; /* "lib/pythonX.Y" */ + wchar_t argv0_path[MAXPATHLEN+1]; + wchar_t zip_path[MAXPATHLEN+1]; /* ".../lib/pythonXY.zip" */ + + int prefix_found; /* found platform independent libraries? */ + int exec_prefix_found; /* found the platform dependent libraries? */ +} PyCalculatePath; + +static const wchar_t delimiter[2] = {DELIM, '\0'}; +static const wchar_t separator[2] = {SEP, '\0'}; +static PyPathConfig path_config = {.module_search_path = NULL}; -/* Get file status. Encode the path to the locale encoding. */ +/* Get file status. Encode the path to the locale encoding. */ static int _Py_wstat(const wchar_t* path, struct stat *buf) { @@ -131,6 +156,7 @@ _Py_wstat(const wchar_t* path, struct stat *buf) return err; } + static void reduce(wchar_t *dir) { @@ -140,14 +166,17 @@ reduce(wchar_t *dir) dir[i] = '\0'; } + static int isfile(wchar_t *filename) /* Is file, not directory */ { struct stat buf; - if (_Py_wstat(filename, &buf) != 0) + if (_Py_wstat(filename, &buf) != 0) { return 0; - if (!S_ISREG(buf.st_mode)) + } + if (!S_ISREG(buf.st_mode)) { return 0; + } return 1; } @@ -155,41 +184,50 @@ isfile(wchar_t *filename) /* Is file, not directory */ static int ismodule(wchar_t *filename) /* Is module -- check for .pyc too */ { - if (isfile(filename)) + if (isfile(filename)) { return 1; + } /* Check for the compiled version of prefix. */ if (wcslen(filename) < MAXPATHLEN) { wcscat(filename, L"c"); - if (isfile(filename)) + if (isfile(filename)) { return 1; + } } return 0; } +/* Is executable file */ static int -isxfile(wchar_t *filename) /* Is executable file */ +isxfile(wchar_t *filename) { struct stat buf; - if (_Py_wstat(filename, &buf) != 0) + if (_Py_wstat(filename, &buf) != 0) { return 0; - if (!S_ISREG(buf.st_mode)) + } + if (!S_ISREG(buf.st_mode)) { return 0; - if ((buf.st_mode & 0111) == 0) + } + if ((buf.st_mode & 0111) == 0) { return 0; + } return 1; } +/* Is directory */ static int -isdir(wchar_t *filename) /* Is directory */ +isdir(wchar_t *filename) { struct stat buf; - if (_Py_wstat(filename, &buf) != 0) + if (_Py_wstat(filename, &buf) != 0) { return 0; - if (!S_ISDIR(buf.st_mode)) + } + if (!S_ISDIR(buf.st_mode)) { return 0; + } return 1; } @@ -207,58 +245,67 @@ static void joinpath(wchar_t *buffer, wchar_t *stuff) { size_t n, k; - if (stuff[0] == SEP) + if (stuff[0] == SEP) { n = 0; + } else { n = wcslen(buffer); - if (n > 0 && buffer[n-1] != SEP && n < MAXPATHLEN) + if (n > 0 && buffer[n-1] != SEP && n < MAXPATHLEN) { buffer[n++] = SEP; + } } - if (n > MAXPATHLEN) + if (n > MAXPATHLEN) { Py_FatalError("buffer overflow in getpath.c's joinpath()"); + } k = wcslen(stuff); - if (n + k > MAXPATHLEN) + if (n + k > MAXPATHLEN) { k = MAXPATHLEN - n; + } wcsncpy(buffer+n, stuff, k); buffer[n+k] = '\0'; } + /* copy_absolute requires that path be allocated at least MAXPATHLEN + 1 bytes and that p be no more than MAXPATHLEN bytes. */ static void copy_absolute(wchar_t *path, wchar_t *p, size_t pathlen) { - if (p[0] == SEP) + if (p[0] == SEP) { wcscpy(path, p); + } else { if (!_Py_wgetcwd(path, pathlen)) { /* unable to get the current directory */ wcscpy(path, p); return; } - if (p[0] == '.' && p[1] == SEP) + if (p[0] == '.' && p[1] == SEP) { p += 2; + } joinpath(path, p); } } + /* absolutize() requires that path be allocated at least MAXPATHLEN+1 bytes. */ static void absolutize(wchar_t *path) { wchar_t buffer[MAXPATHLEN+1]; - if (path[0] == SEP) + if (path[0] == SEP) { return; + } copy_absolute(buffer, path, MAXPATHLEN+1); wcscpy(path, buffer); } + /* search for a prefix value in an environment file. If found, copy it to the provided buffer, which is expected to be no more than MAXPATHLEN bytes long. */ - static int find_env_config_value(FILE * env_file, const wchar_t * key, wchar_t * value) { @@ -272,15 +319,18 @@ find_env_config_value(FILE * env_file, const wchar_t * key, wchar_t * value) PyObject * decoded; int n; - if (p == NULL) + if (p == NULL) { break; + } n = strlen(p); if (p[n - 1] != '\n') { /* line has overflowed - bail */ break; } - if (p[0] == '#') /* Comment - skip */ + if (p[0] == '#') { + /* Comment - skip */ continue; + } decoded = PyUnicode_DecodeUTF8(buffer, n, "surrogateescape"); if (decoded != NULL) { Py_ssize_t k; @@ -307,106 +357,151 @@ find_env_config_value(FILE * env_file, const wchar_t * key, wchar_t * value) return result; } + /* search_for_prefix requires that argv0_path be no more than MAXPATHLEN bytes long. */ static int -search_for_prefix(wchar_t *argv0_path, wchar_t *home, wchar_t *_prefix, - wchar_t *lib_python) +search_for_prefix(PyCalculatePath *calculate, PyPathConfig *config) { size_t n; wchar_t *vpath; /* If PYTHONHOME is set, we believe it unconditionally */ - if (home) { - wchar_t *delim; - wcsncpy(prefix, home, MAXPATHLEN); - prefix[MAXPATHLEN] = L'\0'; - delim = wcschr(prefix, DELIM); - if (delim) + if (calculate->home) { + wcsncpy(config->prefix, calculate->home, MAXPATHLEN); + config->prefix[MAXPATHLEN] = L'\0'; + wchar_t *delim = wcschr(config->prefix, DELIM); + if (delim) { *delim = L'\0'; - joinpath(prefix, lib_python); - joinpath(prefix, LANDMARK); + } + joinpath(config->prefix, calculate->lib_python); + joinpath(config->prefix, LANDMARK); return 1; } /* Check to see if argv[0] is in the build directory */ - wcsncpy(prefix, argv0_path, MAXPATHLEN); - prefix[MAXPATHLEN] = L'\0'; - joinpath(prefix, L"Modules/Setup"); - if (isfile(prefix)) { + wcsncpy(config->prefix, calculate->argv0_path, MAXPATHLEN); + config->prefix[MAXPATHLEN] = L'\0'; + joinpath(config->prefix, L"Modules/Setup"); + if (isfile(config->prefix)) { /* Check VPATH to see if argv0_path is in the build directory. */ vpath = Py_DecodeLocale(VPATH, NULL); if (vpath != NULL) { - wcsncpy(prefix, argv0_path, MAXPATHLEN); - prefix[MAXPATHLEN] = L'\0'; - joinpath(prefix, vpath); + wcsncpy(config->prefix, calculate->argv0_path, MAXPATHLEN); + config->prefix[MAXPATHLEN] = L'\0'; + joinpath(config->prefix, vpath); PyMem_RawFree(vpath); - joinpath(prefix, L"Lib"); - joinpath(prefix, LANDMARK); - if (ismodule(prefix)) + joinpath(config->prefix, L"Lib"); + joinpath(config->prefix, LANDMARK); + if (ismodule(config->prefix)) { return -1; + } } } /* Search from argv0_path, until root is found */ - copy_absolute(prefix, argv0_path, MAXPATHLEN+1); + copy_absolute(config->prefix, calculate->argv0_path, MAXPATHLEN+1); do { - n = wcslen(prefix); - joinpath(prefix, lib_python); - joinpath(prefix, LANDMARK); - if (ismodule(prefix)) + n = wcslen(config->prefix); + joinpath(config->prefix, calculate->lib_python); + joinpath(config->prefix, LANDMARK); + if (ismodule(config->prefix)) { return 1; - prefix[n] = L'\0'; - reduce(prefix); - } while (prefix[0]); + } + config->prefix[n] = L'\0'; + reduce(config->prefix); + } while (config->prefix[0]); /* Look at configure's PREFIX */ - wcsncpy(prefix, _prefix, MAXPATHLEN); - prefix[MAXPATHLEN] = L'\0'; - joinpath(prefix, lib_python); - joinpath(prefix, LANDMARK); - if (ismodule(prefix)) + wcsncpy(config->prefix, calculate->prefix, MAXPATHLEN); + config->prefix[MAXPATHLEN] = L'\0'; + joinpath(config->prefix, calculate->lib_python); + joinpath(config->prefix, LANDMARK); + if (ismodule(config->prefix)) { return 1; + } /* Fail */ return 0; } +static void +calculate_prefix(PyCalculatePath *calculate, PyPathConfig *config) +{ + calculate->prefix_found = search_for_prefix(calculate, config); + if (!calculate->prefix_found) { + if (!Py_FrozenFlag) { + fprintf(stderr, + "Could not find platform independent libraries \n"); + } + wcsncpy(config->prefix, calculate->prefix, MAXPATHLEN); + joinpath(config->prefix, calculate->lib_python); + } + else { + reduce(config->prefix); + } +} + + +static void +calculate_reduce_prefix(PyCalculatePath *calculate, PyPathConfig *config) +{ + /* Reduce prefix and exec_prefix to their essence, + * e.g. /usr/local/lib/python1.5 is reduced to /usr/local. + * If we're loading relative to the build directory, + * return the compiled-in defaults instead. + */ + if (calculate->prefix_found > 0) { + reduce(config->prefix); + reduce(config->prefix); + /* The prefix is the root directory, but reduce() chopped + * off the "/". */ + if (!config->prefix[0]) { + wcscpy(config->prefix, separator); + } + } + else { + wcsncpy(config->prefix, calculate->prefix, MAXPATHLEN); + } +} + + /* search_for_exec_prefix requires that argv0_path be no more than MAXPATHLEN bytes long. */ static int -search_for_exec_prefix(wchar_t *argv0_path, wchar_t *home, - wchar_t *_exec_prefix, wchar_t *lib_python) +search_for_exec_prefix(PyCalculatePath *calculate, PyPathConfig *config) { size_t n; /* If PYTHONHOME is set, we believe it unconditionally */ - if (home) { - wchar_t *delim; - delim = wcschr(home, DELIM); - if (delim) - wcsncpy(exec_prefix, delim+1, MAXPATHLEN); - else - wcsncpy(exec_prefix, home, MAXPATHLEN); - exec_prefix[MAXPATHLEN] = L'\0'; - joinpath(exec_prefix, lib_python); - joinpath(exec_prefix, L"lib-dynload"); + if (calculate->home) { + wchar_t *delim = wcschr(calculate->home, DELIM); + if (delim) { + wcsncpy(config->exec_prefix, delim+1, MAXPATHLEN); + } + else { + wcsncpy(config->exec_prefix, calculate->home, MAXPATHLEN); + } + config->exec_prefix[MAXPATHLEN] = L'\0'; + joinpath(config->exec_prefix, calculate->lib_python); + joinpath(config->exec_prefix, L"lib-dynload"); return 1; } /* Check to see if argv[0] is in the build directory. "pybuilddir.txt" is written by setup.py and contains the relative path to the location of shared library modules. */ - wcsncpy(exec_prefix, argv0_path, MAXPATHLEN); - exec_prefix[MAXPATHLEN] = L'\0'; - joinpath(exec_prefix, L"pybuilddir.txt"); - if (isfile(exec_prefix)) { - FILE *f = _Py_wfopen(exec_prefix, L"rb"); - if (f == NULL) + wcsncpy(config->exec_prefix, calculate->argv0_path, MAXPATHLEN); + config->exec_prefix[MAXPATHLEN] = L'\0'; + joinpath(config->exec_prefix, L"pybuilddir.txt"); + if (isfile(config->exec_prefix)) { + FILE *f = _Py_wfopen(config->exec_prefix, L"rb"); + if (f == NULL) { errno = 0; + } else { char buf[MAXPATHLEN+1]; PyObject *decoded; @@ -422,9 +517,9 @@ search_for_exec_prefix(wchar_t *argv0_path, wchar_t *home, Py_DECREF(decoded); if (k >= 0) { rel_builddir_path[k] = L'\0'; - wcsncpy(exec_prefix, argv0_path, MAXPATHLEN); - exec_prefix[MAXPATHLEN] = L'\0'; - joinpath(exec_prefix, rel_builddir_path); + wcsncpy(config->exec_prefix, calculate->argv0_path, MAXPATHLEN); + config->exec_prefix[MAXPATHLEN] = L'\0'; + joinpath(config->exec_prefix, rel_builddir_path); return -1; } } @@ -432,87 +527,85 @@ search_for_exec_prefix(wchar_t *argv0_path, wchar_t *home, } /* Search from argv0_path, until root is found */ - copy_absolute(exec_prefix, argv0_path, MAXPATHLEN+1); + copy_absolute(config->exec_prefix, calculate->argv0_path, MAXPATHLEN+1); do { - n = wcslen(exec_prefix); - joinpath(exec_prefix, lib_python); - joinpath(exec_prefix, L"lib-dynload"); - if (isdir(exec_prefix)) + n = wcslen(config->exec_prefix); + joinpath(config->exec_prefix, calculate->lib_python); + joinpath(config->exec_prefix, L"lib-dynload"); + if (isdir(config->exec_prefix)) { return 1; - exec_prefix[n] = L'\0'; - reduce(exec_prefix); - } while (exec_prefix[0]); + } + config->exec_prefix[n] = L'\0'; + reduce(config->exec_prefix); + } while (config->exec_prefix[0]); /* Look at configure's EXEC_PREFIX */ - wcsncpy(exec_prefix, _exec_prefix, MAXPATHLEN); - exec_prefix[MAXPATHLEN] = L'\0'; - joinpath(exec_prefix, lib_python); - joinpath(exec_prefix, L"lib-dynload"); - if (isdir(exec_prefix)) + wcsncpy(config->exec_prefix, calculate->exec_prefix, MAXPATHLEN); + config->exec_prefix[MAXPATHLEN] = L'\0'; + joinpath(config->exec_prefix, calculate->lib_python); + joinpath(config->exec_prefix, L"lib-dynload"); + if (isdir(config->exec_prefix)) { return 1; + } /* Fail */ return 0; } + static void -calculate_path(const _PyMainInterpreterConfig *config) +calculate_exec_prefix(PyCalculatePath *calculate, PyPathConfig *config) { - extern wchar_t *Py_GetProgramName(void); - - static const wchar_t delimiter[2] = {DELIM, '\0'}; - static const wchar_t separator[2] = {SEP, '\0'}; - wchar_t *home = _Py_GetPythonHomeWithConfig(config); - char *_path = getenv("PATH"); - wchar_t *path_buffer = NULL; - wchar_t *path = NULL; - wchar_t *prog = Py_GetProgramName(); - wchar_t argv0_path[MAXPATHLEN+1]; - wchar_t zip_path[MAXPATHLEN+1]; - int pfound, efound; /* 1 if found; -1 if found build directory */ - wchar_t *buf; - size_t bufsz; - size_t prefixsz; - wchar_t *defpath; -#ifdef WITH_NEXT_FRAMEWORK - NSModule pythonModule; - const char* modPath; -#endif -#ifdef __APPLE__ -#if MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_4 - uint32_t nsexeclength = MAXPATHLEN; -#else - unsigned long nsexeclength = MAXPATHLEN; -#endif - char execpath[MAXPATHLEN+1]; -#endif - wchar_t *_pythonpath, *_prefix, *_exec_prefix; - wchar_t *lib_python; + calculate->exec_prefix_found = search_for_exec_prefix(calculate, config); + if (!calculate->exec_prefix_found) { + if (!Py_FrozenFlag) { + fprintf(stderr, + "Could not find platform dependent libraries \n"); + } + wcsncpy(config->exec_prefix, calculate->exec_prefix, MAXPATHLEN); + joinpath(config->exec_prefix, L"lib/lib-dynload"); + } + /* If we found EXEC_PREFIX do *not* reduce it! (Yet.) */ +} - _pythonpath = Py_DecodeLocale(PYTHONPATH, NULL); - _prefix = Py_DecodeLocale(PREFIX, NULL); - _exec_prefix = Py_DecodeLocale(EXEC_PREFIX, NULL); - lib_python = Py_DecodeLocale("lib/python" VERSION, NULL); - if (!_pythonpath || !_prefix || !_exec_prefix || !lib_python) { - Py_FatalError( - "Unable to decode path variables in getpath.c: " - "memory error"); +static void +calculate_reduce_exec_prefix(PyCalculatePath *calculate, PyPathConfig *config) +{ + if (calculate->exec_prefix_found > 0) { + reduce(config->exec_prefix); + reduce(config->exec_prefix); + reduce(config->exec_prefix); + if (!config->exec_prefix[0]) { + wcscpy(config->exec_prefix, separator); + } } - - if (_path) { - path_buffer = Py_DecodeLocale(_path, NULL); - path = path_buffer; + else { + wcsncpy(config->exec_prefix, calculate->exec_prefix, MAXPATHLEN); } +} + +static void +calculate_progpath(PyCalculatePath *calculate, PyPathConfig *config) +{ /* If there is no slash in the argv0 path, then we have to * assume python is on the user's $PATH, since there's no * other way to find a directory to start the search from. If * $PATH isn't exported, you lose. */ - if (wcschr(prog, SEP)) - wcsncpy(progpath, prog, MAXPATHLEN); + if (wcschr(calculate->prog, SEP)) { + wcsncpy(config->progpath, calculate->prog, MAXPATHLEN); + } + #ifdef __APPLE__ +#if MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_4 + uint32_t nsexeclength = MAXPATHLEN; +#else + unsigned long nsexeclength = MAXPATHLEN; +#endif + char execpath[MAXPATHLEN+1]; + /* On Mac OS X, if a script uses an interpreter of the form * "#!/opt/python2.3/bin/python", the kernel only passes "python" * as argv[0], which falls through to the $PATH search below. @@ -524,47 +617,60 @@ calculate_path(const _PyMainInterpreterConfig *config) * absolutize() should help us out below */ else if(0 == _NSGetExecutablePath(execpath, &nsexeclength) && execpath[0] == SEP) { - size_t r = mbstowcs(progpath, execpath, MAXPATHLEN+1); + size_t r = mbstowcs(config->progpath, execpath, MAXPATHLEN+1); if (r == (size_t)-1 || r > MAXPATHLEN) { /* Could not convert execpath, or it's too long. */ - progpath[0] = '\0'; + config->progpath[0] = '\0'; } } #endif /* __APPLE__ */ - else if (path) { + else if (calculate->path_env) { + wchar_t *path = calculate->path_env; while (1) { wchar_t *delim = wcschr(path, DELIM); if (delim) { size_t len = delim - path; - if (len > MAXPATHLEN) + if (len > MAXPATHLEN) { len = MAXPATHLEN; - wcsncpy(progpath, path, len); - *(progpath + len) = '\0'; + } + wcsncpy(config->progpath, path, len); + *(config->progpath + len) = '\0'; + } + else { + wcsncpy(config->progpath, path, MAXPATHLEN); } - else - wcsncpy(progpath, path, MAXPATHLEN); - joinpath(progpath, prog); - if (isxfile(progpath)) + joinpath(config->progpath, calculate->prog); + if (isxfile(config->progpath)) { break; + } if (!delim) { - progpath[0] = L'\0'; + config->progpath[0] = L'\0'; break; } path = delim + 1; } } - else - progpath[0] = '\0'; - PyMem_RawFree(path_buffer); - if (progpath[0] != SEP && progpath[0] != '\0') - absolutize(progpath); - wcsncpy(argv0_path, progpath, MAXPATHLEN); - argv0_path[MAXPATHLEN] = '\0'; + else { + config->progpath[0] = '\0'; + } + if (config->progpath[0] != SEP && config->progpath[0] != '\0') { + absolutize(config->progpath); + } +} + + +static void +calculate_argv0_path(PyCalculatePath *calculate, PyPathConfig *config) +{ + wcsncpy(calculate->argv0_path, config->progpath, MAXPATHLEN); + calculate->argv0_path[MAXPATHLEN] = '\0'; #ifdef WITH_NEXT_FRAMEWORK + NSModule pythonModule; + /* On Mac OS X we have a special case if we're running from a framework. ** This is because the python home should be set relative to the library, ** which is in the framework, not relative to the executable, which may @@ -572,7 +678,7 @@ calculate_path(const _PyMainInterpreterConfig *config) */ pythonModule = NSModuleForSymbol(NSLookupAndBindSymbol("_Py_Initialize")); /* Use dylib functions to find out where the framework was loaded from */ - modPath = NSLibraryNameForModule(pythonModule); + const char* modPath = NSLibraryNameForModule(pythonModule); if (modPath != NULL) { /* We're in a framework. */ /* See if we might be in the build directory. The framework in the @@ -587,153 +693,132 @@ calculate_path(const _PyMainInterpreterConfig *config) Py_FatalError("Cannot decode framework location"); } - wcsncpy(argv0_path, wbuf, MAXPATHLEN); - reduce(argv0_path); - joinpath(argv0_path, lib_python); - joinpath(argv0_path, LANDMARK); - if (!ismodule(argv0_path)) { + wcsncpy(calculate->argv0_path, wbuf, MAXPATHLEN); + reduce(calculate->argv0_path); + joinpath(calculate->argv0_path, calculate->lib_python); + joinpath(calculate->argv0_path, LANDMARK); + if (!ismodule(calculate->argv0_path)) { /* We are in the build directory so use the name of the executable - we know that the absolute path is passed */ - wcsncpy(argv0_path, progpath, MAXPATHLEN); + wcsncpy(calculate->argv0_path, config->progpath, MAXPATHLEN); } else { /* Use the location of the library as the progpath */ - wcsncpy(argv0_path, wbuf, MAXPATHLEN); + wcsncpy(calculate->argv0_path, wbuf, MAXPATHLEN); } PyMem_RawFree(wbuf); } #endif #if HAVE_READLINK - { - wchar_t tmpbuffer[MAXPATHLEN+1]; - int linklen = _Py_wreadlink(progpath, tmpbuffer, MAXPATHLEN); - while (linklen != -1) { - if (tmpbuffer[0] == SEP) - /* tmpbuffer should never be longer than MAXPATHLEN, - but extra check does not hurt */ - wcsncpy(argv0_path, tmpbuffer, MAXPATHLEN); - else { - /* Interpret relative to progpath */ - reduce(argv0_path); - joinpath(argv0_path, tmpbuffer); - } - linklen = _Py_wreadlink(argv0_path, tmpbuffer, MAXPATHLEN); + wchar_t tmpbuffer[MAXPATHLEN+1]; + int linklen = _Py_wreadlink(config->progpath, tmpbuffer, MAXPATHLEN); + while (linklen != -1) { + if (tmpbuffer[0] == SEP) { + /* tmpbuffer should never be longer than MAXPATHLEN, + but extra check does not hurt */ + wcsncpy(calculate->argv0_path, tmpbuffer, MAXPATHLEN); + } + else { + /* Interpret relative to progpath */ + reduce(calculate->argv0_path); + joinpath(calculate->argv0_path, tmpbuffer); } + linklen = _Py_wreadlink(calculate->argv0_path, tmpbuffer, MAXPATHLEN); } #endif /* HAVE_READLINK */ - reduce(argv0_path); + reduce(calculate->argv0_path); /* At this point, argv0_path is guaranteed to be less than - MAXPATHLEN bytes long. - */ + MAXPATHLEN bytes long. */ +} - /* Search for an environment configuration file, first in the - executable's directory and then in the parent directory. - If found, open it for use when searching for prefixes. - */ - { - wchar_t tmpbuffer[MAXPATHLEN+1]; - wchar_t *env_cfg = L"pyvenv.cfg"; - FILE * env_file = NULL; +/* Search for an "pyvenv.cfg" environment configuration file, first in the + executable's directory and then in the parent directory. + If found, open it for use when searching for prefixes. +*/ +static void +calculate_read_pyenv(PyCalculatePath *calculate) +{ + wchar_t tmpbuffer[MAXPATHLEN+1]; + wchar_t *env_cfg = L"pyvenv.cfg"; + FILE *env_file; + + wcscpy(tmpbuffer, calculate->argv0_path); - wcscpy(tmpbuffer, argv0_path); + joinpath(tmpbuffer, env_cfg); + env_file = _Py_wfopen(tmpbuffer, L"r"); + if (env_file == NULL) { + errno = 0; + reduce(tmpbuffer); + reduce(tmpbuffer); joinpath(tmpbuffer, env_cfg); + env_file = _Py_wfopen(tmpbuffer, L"r"); if (env_file == NULL) { errno = 0; - reduce(tmpbuffer); - reduce(tmpbuffer); - joinpath(tmpbuffer, env_cfg); - env_file = _Py_wfopen(tmpbuffer, L"r"); - if (env_file == NULL) { - errno = 0; - } - } - if (env_file != NULL) { - /* Look for a 'home' variable and set argv0_path to it, if found */ - if (find_env_config_value(env_file, L"home", tmpbuffer)) { - wcscpy(argv0_path, tmpbuffer); - } - fclose(env_file); - env_file = NULL; } } - pfound = search_for_prefix(argv0_path, home, _prefix, lib_python); - if (!pfound) { - if (!Py_FrozenFlag) - fprintf(stderr, - "Could not find platform independent libraries \n"); - wcsncpy(prefix, _prefix, MAXPATHLEN); - joinpath(prefix, lib_python); - } - else - reduce(prefix); - - wcsncpy(zip_path, prefix, MAXPATHLEN); - zip_path[MAXPATHLEN] = L'\0'; - if (pfound > 0) { /* Use the reduced prefix returned by Py_GetPrefix() */ - reduce(zip_path); - reduce(zip_path); - } - else - wcsncpy(zip_path, _prefix, MAXPATHLEN); - joinpath(zip_path, L"lib/python00.zip"); - bufsz = wcslen(zip_path); /* Replace "00" with version */ - zip_path[bufsz - 6] = VERSION[0]; - zip_path[bufsz - 5] = VERSION[2]; - - efound = search_for_exec_prefix(argv0_path, home, - _exec_prefix, lib_python); - if (!efound) { - if (!Py_FrozenFlag) - fprintf(stderr, - "Could not find platform dependent libraries \n"); - wcsncpy(exec_prefix, _exec_prefix, MAXPATHLEN); - joinpath(exec_prefix, L"lib/lib-dynload"); + if (env_file == NULL) { + return; } - /* If we found EXEC_PREFIX do *not* reduce it! (Yet.) */ - if ((!pfound || !efound) && !Py_FrozenFlag) - fprintf(stderr, - "Consider setting $PYTHONHOME to [:]\n"); + /* Look for a 'home' variable and set argv0_path to it, if found */ + if (find_env_config_value(env_file, L"home", tmpbuffer)) { + wcscpy(calculate->argv0_path, tmpbuffer); + } + fclose(env_file); +} - /* Calculate size of return buffer. - */ - bufsz = 0; - wchar_t *env_path = NULL; - if (config) { - if (config->module_search_path_env) { - bufsz += wcslen(config->module_search_path_env) + 1; - } +static void +calculate_zip_path(PyCalculatePath *calculate, PyPathConfig *config) +{ + wcsncpy(calculate->zip_path, config->prefix, MAXPATHLEN); + calculate->zip_path[MAXPATHLEN] = L'\0'; + + if (calculate->prefix_found > 0) { + /* Use the reduced prefix returned by Py_GetPrefix() */ + reduce(calculate->zip_path); + reduce(calculate->zip_path); } else { - char *env_pathb = Py_GETENV("PYTHONPATH"); - if (env_pathb && env_pathb[0] != '\0') { - size_t env_path_len; - env_path = Py_DecodeLocale(env_pathb, &env_path_len); - /* FIXME: handle decoding and memory error */ - if (env_path != NULL) { - bufsz += env_path_len + 1; - } - } + wcsncpy(calculate->zip_path, calculate->prefix, MAXPATHLEN); } + joinpath(calculate->zip_path, L"lib/python00.zip"); + + /* Replace "00" with version */ + size_t bufsz = wcslen(calculate->zip_path); + calculate->zip_path[bufsz - 6] = VERSION[0]; + calculate->zip_path[bufsz - 5] = VERSION[2]; +} - defpath = _pythonpath; - prefixsz = wcslen(prefix) + 1; + +static wchar_t * +calculate_module_search_path(PyCalculatePath *calculate, PyPathConfig *config) +{ + /* Calculate size of return buffer */ + size_t bufsz = 0; + if (calculate->module_search_path_env != NULL) { + bufsz += wcslen(calculate->module_search_path_env) + 1; + } + + wchar_t *defpath = calculate->pythonpath; + size_t prefixsz = wcslen(config->prefix) + 1; while (1) { wchar_t *delim = wcschr(defpath, DELIM); - if (defpath[0] != SEP) + if (defpath[0] != SEP) { /* Paths are relative to prefix */ bufsz += prefixsz; + } - if (delim) + if (delim) { bufsz += delim - defpath + 1; + } else { bufsz += wcslen(defpath) + 1; break; @@ -741,46 +826,40 @@ calculate_path(const _PyMainInterpreterConfig *config) defpath = delim + 1; } - bufsz += wcslen(zip_path) + 1; - bufsz += wcslen(exec_prefix) + 1; + bufsz += wcslen(calculate->zip_path) + 1; + bufsz += wcslen(config->exec_prefix) + 1; - buf = PyMem_RawMalloc(bufsz * sizeof(wchar_t)); + /* Allocate the buffer */ + wchar_t *buf = PyMem_RawMalloc(bufsz * sizeof(wchar_t)); if (buf == NULL) { Py_FatalError( "Not enough memory for dynamic PYTHONPATH"); } + buf[0] = '\0'; /* Run-time value of $PYTHONPATH goes first */ - buf[0] = '\0'; - if (config) { - if (config->module_search_path_env) { - wcscpy(buf, config->module_search_path_env); - wcscat(buf, delimiter); - } - } - else { - if (env_path) { - wcscpy(buf, env_path); - wcscat(buf, delimiter); - } + if (calculate->module_search_path_env) { + wcscpy(buf, calculate->module_search_path_env); + wcscat(buf, delimiter); } - PyMem_RawFree(env_path); /* Next is the default zip path */ - wcscat(buf, zip_path); + wcscat(buf, calculate->zip_path); wcscat(buf, delimiter); /* Next goes merge of compile-time $PYTHONPATH with * dynamically located prefix. */ - defpath = _pythonpath; + defpath = calculate->pythonpath; while (1) { wchar_t *delim = wcschr(defpath, DELIM); if (defpath[0] != SEP) { - wcscat(buf, prefix); - if (prefixsz >= 2 && prefix[prefixsz - 2] != SEP && - defpath[0] != (delim ? DELIM : L'\0')) { /* not empty */ + wcscat(buf, config->prefix); + if (prefixsz >= 2 && config->prefix[prefixsz - 2] != SEP && + defpath[0] != (delim ? DELIM : L'\0')) + { + /* not empty */ wcscat(buf, separator); } } @@ -800,41 +879,130 @@ calculate_path(const _PyMainInterpreterConfig *config) wcscat(buf, delimiter); /* Finally, on goes the directory for dynamic-load modules */ - wcscat(buf, exec_prefix); + wcscat(buf, config->exec_prefix); - /* And publish the results */ - module_search_path = buf; + return buf; +} - /* Reduce prefix and exec_prefix to their essence, - * e.g. /usr/local/lib/python1.5 is reduced to /usr/local. - * If we're loading relative to the build directory, - * return the compiled-in defaults instead. - */ - if (pfound > 0) { - reduce(prefix); - reduce(prefix); - /* The prefix is the root directory, but reduce() chopped - * off the "/". */ - if (!prefix[0]) - wcscpy(prefix, separator); + +#define DECODE_FAILED(NAME, LEN) \ + ((LEN) == (size_t)-2) \ + ? _Py_INIT_ERR("failed to decode " #NAME) \ + : _Py_INIT_NO_MEMORY() + + +static _PyInitError +calculate_init(PyCalculatePath *calculate, + const _PyMainInterpreterConfig *main_config) +{ + _PyInitError err; + + err = _Py_GetPythonHomeWithConfig(main_config, &calculate->home); + if (_Py_INIT_FAILED(err)) { + return err; + } + + size_t len; + char *path = getenv("PATH"); + if (path) { + calculate->path_env = Py_DecodeLocale(path, &len); + if (!calculate->path_env) { + return DECODE_FAILED("PATH environment variable", len); + } + } + + calculate->prog = Py_GetProgramName(); + + calculate->pythonpath = Py_DecodeLocale(PYTHONPATH, &len); + if (!calculate->pythonpath) { + return DECODE_FAILED("PYTHONPATH define", len); + } + calculate->prefix = Py_DecodeLocale(PREFIX, &len); + if (!calculate->prefix) { + return DECODE_FAILED("PREFIX define", len); + } + calculate->exec_prefix = Py_DecodeLocale(EXEC_PREFIX, &len); + if (!calculate->prefix) { + return DECODE_FAILED("EXEC_PREFIX define", len); + } + calculate->lib_python = Py_DecodeLocale("lib/python" VERSION, &len); + if (!calculate->lib_python) { + return DECODE_FAILED("EXEC_PREFIX define", len); } - else - wcsncpy(prefix, _prefix, MAXPATHLEN); - if (efound > 0) { - reduce(exec_prefix); - reduce(exec_prefix); - reduce(exec_prefix); - if (!exec_prefix[0]) - wcscpy(exec_prefix, separator); + calculate->module_search_path_env = NULL; + if (main_config) { + if (main_config->module_search_path_env) { + calculate->module_search_path_env = main_config->module_search_path_env; + } + + } + else { + char *pythonpath = Py_GETENV("PYTHONPATH"); + if (pythonpath && pythonpath[0] != '\0') { + calculate->module_search_path_buffer = Py_DecodeLocale(pythonpath, &len); + if (!calculate->module_search_path_buffer) { + return DECODE_FAILED("PYTHONPATH environment variable", len); + } + calculate->module_search_path_env = calculate->module_search_path_buffer; + } } - else - wcsncpy(exec_prefix, _exec_prefix, MAXPATHLEN); + return _Py_INIT_OK(); +} + - PyMem_RawFree(_pythonpath); - PyMem_RawFree(_prefix); - PyMem_RawFree(_exec_prefix); - PyMem_RawFree(lib_python); +static void +calculate_free(PyCalculatePath *calculate) +{ + PyMem_RawFree(calculate->pythonpath); + PyMem_RawFree(calculate->prefix); + PyMem_RawFree(calculate->exec_prefix); + PyMem_RawFree(calculate->lib_python); + PyMem_RawFree(calculate->path_env); + PyMem_RawFree(calculate->module_search_path_buffer); +} + + +static void +calculate_path_impl(PyCalculatePath *calculate, PyPathConfig *config) +{ + calculate_progpath(calculate, config); + calculate_argv0_path(calculate, config); + calculate_read_pyenv(calculate); + calculate_prefix(calculate, config); + calculate_zip_path(calculate, config); + calculate_exec_prefix(calculate, config); + + if ((!calculate->prefix_found || !calculate->exec_prefix_found) && !Py_FrozenFlag) { + fprintf(stderr, + "Consider setting $PYTHONHOME to [:]\n"); + } + + config->module_search_path = calculate_module_search_path(calculate, config); + calculate_reduce_prefix(calculate, config); + calculate_reduce_exec_prefix(calculate, config); +} + + +static void +calculate_path(const _PyMainInterpreterConfig *main_config) +{ + PyCalculatePath calculate; + memset(&calculate, 0, sizeof(calculate)); + + _PyInitError err = calculate_init(&calculate, main_config); + if (_Py_INIT_FAILED(err)) { + calculate_free(&calculate); + _Py_FatalInitError(err); + } + + PyPathConfig new_path_config; + memset(&new_path_config, 0, sizeof(new_path_config)); + + calculate_path_impl(&calculate, &new_path_config); + path_config = new_path_config; + + calculate_free(&calculate); } @@ -842,63 +1010,74 @@ calculate_path(const _PyMainInterpreterConfig *config) void Py_SetPath(const wchar_t *path) { - if (module_search_path != NULL) { - PyMem_RawFree(module_search_path); - module_search_path = NULL; + if (path_config.module_search_path != NULL) { + PyMem_RawFree(path_config.module_search_path); + path_config.module_search_path = NULL; } - if (path != NULL) { - extern wchar_t *Py_GetProgramName(void); - wchar_t *prog = Py_GetProgramName(); - wcsncpy(progpath, prog, MAXPATHLEN); - exec_prefix[0] = prefix[0] = L'\0'; - module_search_path = PyMem_RawMalloc((wcslen(path) + 1) * sizeof(wchar_t)); - if (module_search_path != NULL) - wcscpy(module_search_path, path); + + if (path == NULL) { + return; + } + + wchar_t *prog = Py_GetProgramName(); + wcsncpy(path_config.progpath, prog, MAXPATHLEN); + path_config.exec_prefix[0] = path_config.prefix[0] = L'\0'; + path_config.module_search_path = PyMem_RawMalloc((wcslen(path) + 1) * sizeof(wchar_t)); + if (path_config.module_search_path != NULL) { + wcscpy(path_config.module_search_path, path); } } + wchar_t * -_Py_GetPathWithConfig(const _PyMainInterpreterConfig *config) +_Py_GetPathWithConfig(const _PyMainInterpreterConfig *main_config) { - if (!module_search_path) { - calculate_path(config); + if (!path_config.module_search_path) { + calculate_path(main_config); } - return module_search_path; + return path_config.module_search_path; } + wchar_t * Py_GetPath(void) { - if (!module_search_path) + if (!path_config.module_search_path) { calculate_path(NULL); - return module_search_path; + } + return path_config.module_search_path; } + wchar_t * Py_GetPrefix(void) { - if (!module_search_path) + if (!path_config.module_search_path) { calculate_path(NULL); - return prefix; + } + return path_config.prefix; } + wchar_t * Py_GetExecPrefix(void) { - if (!module_search_path) + if (!path_config.module_search_path) { calculate_path(NULL); - return exec_prefix; + } + return path_config.exec_prefix; } + wchar_t * Py_GetProgramFullPath(void) { - if (!module_search_path) + if (!path_config.module_search_path) { calculate_path(NULL); - return progpath; + } + return path_config.progpath; } - #ifdef __cplusplus } #endif diff --git a/Modules/main.c b/Modules/main.c index 07e0d2aa85ca49..349d8c3f3239ad 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -36,6 +36,16 @@ extern "C" { #endif +#define SET_DECODE_ERROR(NAME, LEN) \ + do { \ + if ((LEN) == (size_t)-2) { \ + pymain->err = _Py_INIT_ERR("failed to decode " #NAME); \ + } \ + else { \ + pymain->err = _Py_INIT_NO_MEMORY(); \ + } \ + } while (0) + /* For Py_GetArgcArgv(); set by main() */ static wchar_t **orig_argv; static int orig_argc; @@ -417,9 +427,6 @@ typedef struct { .env_warning_options = {0, NULL}} -#define INIT_NO_MEMORY() _Py_INIT_ERR("memory allocation failed") - - static void pymain_optlist_clear(_Py_OptList *list) { @@ -510,14 +517,14 @@ pymain_wstrdup(_PyMain *pymain, wchar_t *str) { size_t len = wcslen(str) + 1; /* +1 for NUL character */ if (len > (size_t)PY_SSIZE_T_MAX / sizeof(wchar_t)) { - pymain->err = INIT_NO_MEMORY(); + pymain->err = _Py_INIT_NO_MEMORY(); return NULL; } size_t size = len * sizeof(wchar_t); wchar_t *str2 = PyMem_RawMalloc(size); if (str2 == NULL) { - pymain->err = INIT_NO_MEMORY(); + pymain->err = _Py_INIT_NO_MEMORY(); return NULL; } @@ -538,7 +545,7 @@ pymain_optlist_append(_PyMain *pymain, _Py_OptList *list, wchar_t *str) wchar_t **options2 = (wchar_t **)PyMem_RawRealloc(list->options, size); if (options2 == NULL) { PyMem_RawFree(str2); - pymain->err = INIT_NO_MEMORY(); + pymain->err = _Py_INIT_NO_MEMORY(); return -1; } options2[list->len] = str2; @@ -571,7 +578,7 @@ pymain_parse_cmdline_impl(_PyMain *pymain) size_t len = wcslen(_PyOS_optarg) + 1 + 1; wchar_t *command = PyMem_RawMalloc(sizeof(wchar_t) * len); if (command == NULL) { - pymain->err = INIT_NO_MEMORY(); + pymain->err = _Py_INIT_NO_MEMORY(); return -1; } memcpy(command, _PyOS_optarg, len * sizeof(wchar_t)); @@ -717,7 +724,7 @@ pymain_add_xoptions(_PyMain *pymain) for (size_t i=0; i < options->len; i++) { wchar_t *option = options->options[i]; if (_PySys_AddXOptionWithError(option) < 0) { - pymain->err = INIT_NO_MEMORY(); + pymain->err = _Py_INIT_NO_MEMORY(); return -1; } } @@ -748,11 +755,11 @@ pymain_add_warnings_options(_PyMain *pymain) PySys_ResetWarnOptions(); if (pymain_add_warnings_optlist(&pymain->env_warning_options) < 0) { - pymain->err = INIT_NO_MEMORY(); + pymain->err = _Py_INIT_NO_MEMORY(); return -1; } if (pymain_add_warnings_optlist(&pymain->cmdline.warning_options) < 0) { - pymain->err = INIT_NO_MEMORY(); + pymain->err = _Py_INIT_NO_MEMORY(); return -1; } return 0; @@ -801,7 +808,7 @@ pymain_warnings_envvar(_PyMain *pymain) C89 wcstok */ buf = (char *)PyMem_RawMalloc(strlen(p) + 1); if (buf == NULL) { - pymain->err = INIT_NO_MEMORY(); + pymain->err = _Py_INIT_NO_MEMORY(); return -1; } strcpy(buf, p); @@ -811,13 +818,7 @@ pymain_warnings_envvar(_PyMain *pymain) size_t len; wchar_t *warning = Py_DecodeLocale(p, &len); if (warning == NULL) { - if (len == (size_t)-2) { - pymain->err = _Py_INIT_ERR("failed to decode " - "PYTHONWARNINGS"); - } - else { - pymain->err = INIT_NO_MEMORY(); - } + SET_DECODE_ERROR("PYTHONWARNINGS environment variable", len); return -1; } if (pymain_optlist_append(pymain, &pymain->env_warning_options, @@ -902,7 +903,7 @@ pymain_get_program_name(_PyMain *pymain) buffer = PyMem_RawMalloc(len * sizeof(wchar_t)); if (buffer == NULL) { - pymain->err = INIT_NO_MEMORY(); + pymain->err = _Py_INIT_NO_MEMORY(); return -1; } @@ -919,15 +920,8 @@ pymain_get_program_name(_PyMain *pymain) size_t len; wchar_t* wbuf = Py_DecodeLocale(pyvenv_launcher, &len); if (wbuf == NULL) { - if (len == (size_t)-2) { - pymain->err = _Py_INIT_ERR("failed to decode " - "__PYVENV_LAUNCHER__"); - return -1; - } - else { - pymain->err = INIT_NO_MEMORY(); - return -1; - } + SET_DECODE_ERROR("__PYVENV_LAUNCHER__", len); + return -1; } pymain->program_name = wbuf; } @@ -1403,7 +1397,7 @@ pymain_get_env_var_dup(_PyMain *pymain, wchar_t **dest, return -2; } else { - pymain->err = INIT_NO_MEMORY(); + pymain->err = _Py_INIT_NO_MEMORY(); return -1; } } @@ -1421,7 +1415,7 @@ pymain_init_pythonpath(_PyMain *pymain) L"PYTHONPATH", "PYTHONPATH"); if (res < 0) { if (res == -2) { - pymain->err = _Py_INIT_ERR("failed to decode PYTHONPATH"); + SET_DECODE_ERROR("PYTHONPATH", (size_t)-2); } return -1; } @@ -1450,7 +1444,7 @@ pymain_init_pythonhome(_PyMain *pymain) L"PYTHONHOME", "PYTHONHOME"); if (res < 0) { if (res == -2) { - pymain->err = _Py_INIT_ERR("failed to decode PYTHONHOME"); + SET_DECODE_ERROR("PYTHONHOME", (size_t)-2); } return -1; } diff --git a/PC/getpathp.c b/PC/getpathp.c index 4756dc8abbb620..e0cb9a25758496 100644 --- a/PC/getpathp.c +++ b/PC/getpathp.c @@ -116,14 +116,34 @@ #define LANDMARK L"lib\\os.py" #endif -static wchar_t prefix[MAXPATHLEN+1]; -static wchar_t progpath[MAXPATHLEN+1]; -static wchar_t dllpath[MAXPATHLEN+1]; -static wchar_t *module_search_path = NULL; +typedef struct { + wchar_t prefix[MAXPATHLEN+1]; + wchar_t progpath[MAXPATHLEN+1]; + wchar_t dllpath[MAXPATHLEN+1]; + wchar_t *module_search_path; +} PyPathConfig; + +typedef struct { + wchar_t *module_search_path_env; /* PYTHONPATH environment variable */ + wchar_t *path_env; /* PATH environment variable */ + wchar_t *home; /* PYTHONHOME environment variable */ + + /* Registry key "Software\Python\PythonCore\PythonPath" */ + wchar_t *machine_path; /* from HKEY_LOCAL_MACHINE */ + wchar_t *user_path; /* from HKEY_CURRENT_USER */ + + wchar_t *prog; /* Program name */ + wchar_t argv0_path[MAXPATHLEN+1]; + wchar_t zip_path[MAXPATHLEN+1]; +} PyCalculatePath; + +static PyPathConfig path_config = {.module_search_path = NULL}; + +/* determine if "ch" is a separator character */ static int -is_sep(wchar_t ch) /* determine if "ch" is a separator character */ +is_sep(wchar_t ch) { #ifdef ALTSEP return ch == SEP || ch == ALTSEP; @@ -132,28 +152,31 @@ is_sep(wchar_t ch) /* determine if "ch" is a separator character */ #endif } + /* assumes 'dir' null terminated in bounds. Never writes - beyond existing terminator. -*/ + beyond existing terminator. */ static void reduce(wchar_t *dir) { size_t i = wcsnlen_s(dir, MAXPATHLEN+1); - if (i >= MAXPATHLEN+1) + if (i >= MAXPATHLEN+1) { Py_FatalError("buffer overflow in getpathp.c's reduce()"); + } while (i > 0 && !is_sep(dir[i])) --i; dir[i] = '\0'; } + static int change_ext(wchar_t *dest, const wchar_t *src, const wchar_t *ext) { size_t src_len = wcsnlen_s(src, MAXPATHLEN+1); size_t i = src_len; - if (i >= MAXPATHLEN+1) + if (i >= MAXPATHLEN+1) { Py_FatalError("buffer overflow in getpathp.c's reduce()"); + } while (i > 0 && src[i] != '.' && !is_sep(src[i])) --i; @@ -163,11 +186,13 @@ change_ext(wchar_t *dest, const wchar_t *src, const wchar_t *ext) return -1; } - if (is_sep(src[i])) + if (is_sep(src[i])) { i = src_len; + } if (wcsncpy_s(dest, MAXPATHLEN+1, src, i) || - wcscat_s(dest, MAXPATHLEN+1, ext)) { + wcscat_s(dest, MAXPATHLEN+1, ext)) + { dest[0] = '\0'; return -1; } @@ -175,22 +200,25 @@ change_ext(wchar_t *dest, const wchar_t *src, const wchar_t *ext) return 0; } + static int exists(wchar_t *filename) { return GetFileAttributesW(filename) != 0xFFFFFFFF; } -/* Assumes 'filename' MAXPATHLEN+1 bytes long - - may extend 'filename' by one character. -*/ + +/* Is module -- check for .pyc too. + Assumes 'filename' MAXPATHLEN+1 bytes long - + may extend 'filename' by one character. */ static int -ismodule(wchar_t *filename, int update_filename) /* Is module -- check for .pyc too */ +ismodule(wchar_t *filename, int update_filename) { size_t n; - if (exists(filename)) + if (exists(filename)) { return 1; + } /* Check for the compiled version of prefix. */ n = wcsnlen_s(filename, MAXPATHLEN+1); @@ -199,13 +227,15 @@ ismodule(wchar_t *filename, int update_filename) /* Is module -- check for .pyc filename[n] = L'c'; filename[n + 1] = L'\0'; exist = exists(filename); - if (!update_filename) + if (!update_filename) { filename[n] = L'\0'; + } return exist; } return 0; } + /* Add a path component, by appending stuff to buffer. buffer must have at least MAXPATHLEN + 1 bytes allocated, and contain a NUL-terminated string with no more than MAXPATHLEN characters (not counting @@ -217,7 +247,9 @@ ismodule(wchar_t *filename, int update_filename) /* Is module -- check for .pyc */ static int _PathCchCombineEx_Initialized = 0; -typedef HRESULT(__stdcall *PPathCchCombineEx)(PWSTR pszPathOut, size_t cchPathOut, PCWSTR pszPathIn, PCWSTR pszMore, unsigned long dwFlags); +typedef HRESULT(__stdcall *PPathCchCombineEx) (PWSTR pszPathOut, size_t cchPathOut, + PCWSTR pszPathIn, PCWSTR pszMore, + unsigned long dwFlags); static PPathCchCombineEx _PathCchCombineEx; static void @@ -225,28 +257,32 @@ join(wchar_t *buffer, const wchar_t *stuff) { if (_PathCchCombineEx_Initialized == 0) { HMODULE pathapi = LoadLibraryW(L"api-ms-win-core-path-l1-1-0.dll"); - if (pathapi) + if (pathapi) { _PathCchCombineEx = (PPathCchCombineEx)GetProcAddress(pathapi, "PathCchCombineEx"); - else + } + else { _PathCchCombineEx = NULL; + } _PathCchCombineEx_Initialized = 1; } if (_PathCchCombineEx) { - if (FAILED(_PathCchCombineEx(buffer, MAXPATHLEN+1, buffer, stuff, 0))) + if (FAILED(_PathCchCombineEx(buffer, MAXPATHLEN+1, buffer, stuff, 0))) { Py_FatalError("buffer overflow in getpathp.c's join()"); + } } else { - if (!PathCombineW(buffer, buffer, stuff)) + if (!PathCombineW(buffer, buffer, stuff)) { Py_FatalError("buffer overflow in getpathp.c's join()"); + } } } + /* gotlandmark only called by search_for_prefix, which ensures 'prefix' is null terminated in bounds. join() ensures - 'landmark' can not overflow prefix if too long. -*/ + 'landmark' can not overflow prefix if too long. */ static int -gotlandmark(const wchar_t *landmark) +gotlandmark(wchar_t *prefix, const wchar_t *landmark) { int ok; Py_ssize_t n = wcsnlen_s(prefix, MAXPATHLEN); @@ -257,27 +293,29 @@ gotlandmark(const wchar_t *landmark) return ok; } + /* assumes argv0_path is MAXPATHLEN+1 bytes long, already \0 term'd. assumption provided by only caller, calculate_path() */ static int -search_for_prefix(wchar_t *argv0_path, const wchar_t *landmark) +search_for_prefix(wchar_t *prefix, wchar_t *argv0_path, const wchar_t *landmark) { /* Search from argv0_path, until landmark is found */ wcscpy_s(prefix, MAXPATHLEN + 1, argv0_path); do { - if (gotlandmark(landmark)) + if (gotlandmark(prefix, landmark)) { return 1; + } reduce(prefix); } while (prefix[0]); return 0; } + #ifdef Py_ENABLE_SHARED /* a string loaded from the DLL at startup.*/ extern const char *PyWin_DLLVersionString; - /* Load a PYTHONPATH value from the registry. Load from either HKEY_LOCAL_MACHINE or HKEY_CURRENT_USER. @@ -290,7 +328,6 @@ extern const char *PyWin_DLLVersionString; work on Win16, where the buffer sizes werent available in advance. It could be simplied now Win16/Win32s is dead! */ - static wchar_t * getpythonregpath(HKEY keyBase, int skipcore) { @@ -315,7 +352,9 @@ getpythonregpath(HKEY keyBase, int skipcore) sizeof(WCHAR)*(versionLen-1) + sizeof(keySuffix); keyBuf = keyBufPtr = PyMem_RawMalloc(keyBufLen); - if (keyBuf==NULL) goto done; + if (keyBuf==NULL) { + goto done; + } memcpy_s(keyBufPtr, keyBufLen, keyPrefix, sizeof(keyPrefix)-sizeof(WCHAR)); keyBufPtr += Py_ARRAY_LENGTH(keyPrefix) - 1; @@ -329,17 +368,25 @@ getpythonregpath(HKEY keyBase, int skipcore) 0, /* reserved */ KEY_READ, &newKey); - if (rc!=ERROR_SUCCESS) goto done; + if (rc!=ERROR_SUCCESS) { + goto done; + } /* Find out how big our core buffer is, and how many subkeys we have */ rc = RegQueryInfoKey(newKey, NULL, NULL, NULL, &numKeys, NULL, NULL, NULL, NULL, &dataSize, NULL, NULL); - if (rc!=ERROR_SUCCESS) goto done; - if (skipcore) dataSize = 0; /* Only count core ones if we want them! */ + if (rc!=ERROR_SUCCESS) { + goto done; + } + if (skipcore) { + dataSize = 0; /* Only count core ones if we want them! */ + } /* Allocate a temp array of char buffers, so we only need to loop reading the registry once */ ppPaths = PyMem_RawMalloc( sizeof(WCHAR *) * numKeys ); - if (ppPaths==NULL) goto done; + if (ppPaths==NULL) { + goto done; + } memset(ppPaths, 0, sizeof(WCHAR *) * numKeys); /* Loop over all subkeys, allocating a temp sub-buffer. */ for(index=0;indexpath_env; #ifdef Py_ENABLE_SHARED extern HANDLE PyWin_DLLhModule; /* static init of progpath ensures final char remains \0 */ - if (PyWin_DLLhModule) - if (!GetModuleFileNameW(PyWin_DLLhModule, dllpath, MAXPATHLEN)) + if (PyWin_DLLhModule) { + if (!GetModuleFileNameW(PyWin_DLLhModule, dllpath, MAXPATHLEN)) { dllpath[0] = 0; + } + } #else dllpath[0] = 0; #endif - if (GetModuleFileNameW(NULL, progpath, MAXPATHLEN)) + if (GetModuleFileNameW(NULL, progpath, MAXPATHLEN)) { return; - if (prog == NULL || *prog == '\0') - prog = L"python"; + } /* If there is no slash in the argv0 path, then we have to * assume python is on the user's $PATH, since there's no @@ -454,11 +509,13 @@ get_progpath(void) * $PATH isn't exported, you lose. */ #ifdef ALTSEP - if (wcschr(prog, SEP) || wcschr(prog, ALTSEP)) + if (wcschr(calculate->prog, SEP) || wcschr(calculate->prog, ALTSEP)) #else - if (wcschr(prog, SEP)) + if (wcschr(calculate->prog, SEP)) #endif - wcsncpy(progpath, prog, MAXPATHLEN); + { + wcsncpy(progpath, calculate->prog, MAXPATHLEN); + } else if (path) { while (1) { wchar_t *delim = wcschr(path, DELIM); @@ -470,13 +527,15 @@ get_progpath(void) wcsncpy(progpath, path, len); *(progpath + len) = '\0'; } - else + else { wcsncpy(progpath, path, MAXPATHLEN); + } /* join() is safe for MAXPATHLEN+1 size buffer */ - join(progpath, prog); - if (exists(progpath)) + join(progpath, calculate->prog); + if (exists(progpath)) { break; + } if (!delim) { progpath[0] = '\0'; @@ -485,10 +544,12 @@ get_progpath(void) path = delim + 1; } } - else + else { progpath[0] = '\0'; + } } + static int find_env_config_value(FILE * env_file, const wchar_t * key, wchar_t * value) { @@ -502,15 +563,18 @@ find_env_config_value(FILE * env_file, const wchar_t * key, wchar_t * value) PyObject * decoded; size_t n; - if (p == NULL) + if (p == NULL) { break; + } n = strlen(p); if (p[n - 1] != '\n') { /* line has overflowed - bail */ break; } - if (p[0] == '#') /* Comment - skip */ + if (p[0] == '#') { + /* Comment - skip */ continue; + } decoded = PyUnicode_DecodeUTF8(buffer, n, "surrogateescape"); if (decoded != NULL) { Py_ssize_t k; @@ -537,12 +601,14 @@ find_env_config_value(FILE * env_file, const wchar_t * key, wchar_t * value) return result; } -static int + +static wchar_t* read_pth_file(const wchar_t *path, wchar_t *prefix, int *isolated, int *nosite) { FILE *sp_file = _Py_wfopen(path, L"r"); - if (sp_file == NULL) - return -1; + if (sp_file == NULL) { + return NULL; + } wcscpy_s(prefix, MAXPATHLEN+1, path); reduce(prefix); @@ -558,10 +624,12 @@ read_pth_file(const wchar_t *path, wchar_t *prefix, int *isolated, int *nosite) while (!feof(sp_file)) { char line[MAXPATHLEN + 1]; char *p = fgets(line, MAXPATHLEN + 1, sp_file); - if (!p) + if (!p) { break; - if (*p == '\0' || *p == '\r' || *p == '\n' || *p == '#') + } + if (*p == '\0' || *p == '\r' || *p == '\n' || *p == '#') { continue; + } while (*++p) { if (*p == '\r' || *p == '\n') { *p = '\0'; @@ -611,126 +679,176 @@ read_pth_file(const wchar_t *path, wchar_t *prefix, int *isolated, int *nosite) PyMem_RawFree(wline); } - module_search_path = buf; - fclose(sp_file); - return 0; + return buf; error: PyMem_RawFree(buf); fclose(sp_file); - return -1; + return NULL; } -static void -calculate_path(const _PyMainInterpreterConfig *config) +static _PyInitError +calculate_init(PyCalculatePath *calculate, + const _PyMainInterpreterConfig *main_config) { - wchar_t argv0_path[MAXPATHLEN+1]; - wchar_t *buf; - size_t bufsz; - wchar_t *pythonhome = _Py_GetPythonHomeWithConfig(config); - wchar_t *envpath = NULL; - - int skiphome, skipdefault; - wchar_t *machinepath = NULL; - wchar_t *userpath = NULL; - wchar_t zip_path[MAXPATHLEN+1]; + _PyInitError err; - if (config) { - envpath = config->module_search_path_env; + err = _Py_GetPythonHomeWithConfig(main_config, &calculate->home); + if (_Py_INIT_FAILED(err)) { + return err; + } + + if (main_config) { + calculate->module_search_path_env = main_config->module_search_path_env; } else if (!Py_IgnoreEnvironmentFlag) { - envpath = _wgetenv(L"PYTHONPATH"); - if (envpath && *envpath == '\0') - envpath = NULL; + wchar_t *path = _wgetenv(L"PYTHONPATH"); + if (path && *path != '\0') { + calculate->module_search_path_env = path; + } } - get_progpath(); - /* progpath guaranteed \0 terminated in MAXPATH+1 bytes. */ - wcscpy_s(argv0_path, MAXPATHLEN+1, progpath); - reduce(argv0_path); + calculate->path_env = _wgetenv(L"PATH"); - /* Search for a sys.path file */ - { - wchar_t spbuffer[MAXPATHLEN+1]; + wchar_t *prog = Py_GetProgramName(); + if (prog == NULL || *prog == '\0') { + prog = L"python"; + } + calculate->prog = prog; - if ((dllpath[0] && !change_ext(spbuffer, dllpath, L"._pth") && exists(spbuffer)) || - (progpath[0] && !change_ext(spbuffer, progpath, L"._pth") && exists(spbuffer))) { + return _Py_INIT_OK(); +} - if (!read_pth_file(spbuffer, prefix, &Py_IsolatedFlag, &Py_NoSiteFlag)) { - return; - } + +static int +get_pth_filename(wchar_t *spbuffer, PyPathConfig *config) +{ + if (config->dllpath[0]) { + if (!change_ext(spbuffer, config->dllpath, L"._pth") && exists(spbuffer)) { + return 1; + } + } + if (config->progpath[0]) { + if (!change_ext(spbuffer, config->progpath, L"._pth") && exists(spbuffer)) { + return 1; } } + return 0; +} - /* Search for an environment configuration file, first in the - executable's directory and then in the parent directory. - If found, open it for use when searching for prefixes. - */ - { - wchar_t envbuffer[MAXPATHLEN+1]; - wchar_t tmpbuffer[MAXPATHLEN+1]; - const wchar_t *env_cfg = L"pyvenv.cfg"; - FILE * env_file = NULL; +static int +calculate_pth_file(PyPathConfig *config) +{ + wchar_t spbuffer[MAXPATHLEN+1]; + + if (!get_pth_filename(spbuffer, config)) { + return 0; + } + + config->module_search_path = read_pth_file(spbuffer, config->prefix, + &Py_IsolatedFlag, + &Py_NoSiteFlag); + if (!config->module_search_path) { + return 0; + } + return 1; +} + + +/* Search for an environment configuration file, first in the + executable's directory and then in the parent directory. + If found, open it for use when searching for prefixes. +*/ +static void +calculate_pyvenv_file(PyCalculatePath *calculate) +{ + wchar_t envbuffer[MAXPATHLEN+1]; + const wchar_t *env_cfg = L"pyvenv.cfg"; + + wcscpy_s(envbuffer, MAXPATHLEN+1, calculate->argv0_path); + join(envbuffer, env_cfg); - wcscpy_s(envbuffer, MAXPATHLEN+1, argv0_path); + FILE *env_file = _Py_wfopen(envbuffer, L"r"); + if (env_file == NULL) { + errno = 0; + reduce(envbuffer); + reduce(envbuffer); join(envbuffer, env_cfg); env_file = _Py_wfopen(envbuffer, L"r"); if (env_file == NULL) { errno = 0; - reduce(envbuffer); - reduce(envbuffer); - join(envbuffer, env_cfg); - env_file = _Py_wfopen(envbuffer, L"r"); - if (env_file == NULL) { - errno = 0; - } - } - if (env_file != NULL) { - /* Look for a 'home' variable and set argv0_path to it, if found */ - if (find_env_config_value(env_file, L"home", tmpbuffer)) { - wcscpy_s(argv0_path, MAXPATHLEN+1, tmpbuffer); - } - fclose(env_file); - env_file = NULL; } } - /* Calculate zip archive path from DLL or exe path */ - change_ext(zip_path, dllpath[0] ? dllpath : progpath, L".zip"); + if (env_file == NULL) { + return; + } - if (pythonhome == NULL || *pythonhome == '\0') { - if (zip_path[0] && exists(zip_path)) { - wcscpy_s(prefix, MAXPATHLEN+1, zip_path); - reduce(prefix); - pythonhome = prefix; - } else if (search_for_prefix(argv0_path, LANDMARK)) - pythonhome = prefix; - else - pythonhome = NULL; + /* Look for a 'home' variable and set argv0_path to it, if found */ + wchar_t tmpbuffer[MAXPATHLEN+1]; + if (find_env_config_value(env_file, L"home", tmpbuffer)) { + wcscpy_s(calculate->argv0_path, MAXPATHLEN+1, tmpbuffer); } - else - wcscpy_s(prefix, MAXPATHLEN+1, pythonhome); + fclose(env_file); +} - skiphome = pythonhome==NULL ? 0 : 1; +static void +calculate_path_impl(PyCalculatePath *calculate, PyPathConfig *config, + const _PyMainInterpreterConfig *main_config) +{ + get_progpath(calculate, config->progpath, config->dllpath); + /* progpath guaranteed \0 terminated in MAXPATH+1 bytes. */ + wcscpy_s(calculate->argv0_path, MAXPATHLEN+1, config->progpath); + reduce(calculate->argv0_path); + + /* Search for a sys.path file */ + if (calculate_pth_file(config)) { + return; + } + + calculate_pyvenv_file(calculate); + + /* Calculate zip archive path from DLL or exe path */ + change_ext(calculate->zip_path, + config->dllpath[0] ? config->dllpath : config->progpath, + L".zip"); + + if (calculate->home == NULL || *calculate->home == '\0') { + if (calculate->zip_path[0] && exists(calculate->zip_path)) { + wcscpy_s(config->prefix, MAXPATHLEN+1, calculate->zip_path); + reduce(config->prefix); + calculate->home = config->prefix; + } else if (search_for_prefix(config->prefix, calculate->argv0_path, LANDMARK)) { + calculate->home = config->prefix; + } + else { + calculate->home = NULL; + } + } + else { + wcscpy_s(config->prefix, MAXPATHLEN+1, calculate->home); + } + + int skiphome = calculate->home==NULL ? 0 : 1; #ifdef Py_ENABLE_SHARED - machinepath = getpythonregpath(HKEY_LOCAL_MACHINE, skiphome); - userpath = getpythonregpath(HKEY_CURRENT_USER, skiphome); + calculate->machine_path = getpythonregpath(HKEY_LOCAL_MACHINE, skiphome); + calculate->user_path = getpythonregpath(HKEY_CURRENT_USER, skiphome); #endif /* We only use the default relative PYTHONPATH if we haven't anything better to use! */ - skipdefault = envpath!=NULL || pythonhome!=NULL || \ - machinepath!=NULL || userpath!=NULL; + int skipdefault = (calculate->module_search_path_env!=NULL || calculate->home!=NULL || \ + calculate->machine_path!=NULL || calculate->user_path!=NULL); /* We need to construct a path from the following parts. (1) the PYTHONPATH environment variable, if set; (2) for Win32, the zip archive file path; - (3) for Win32, the machinepath and userpath, if set; + (3) for Win32, the machine_path and user_path, if set; (4) the PYTHONPATH config macro, with the leading "." - of each component replaced with pythonhome, if set; + of each component replaced with home, if set; (5) the directory containing the executable (argv0_path). The length calculation calculates #4 first. Extra rules: @@ -739,74 +857,80 @@ calculate_path(const _PyMainInterpreterConfig *config) */ /* Calculate size of return buffer */ - if (pythonhome != NULL) { + size_t bufsz = 0; + if (calculate->home != NULL) { wchar_t *p; bufsz = 1; for (p = PYTHONPATH; *p; p++) { - if (*p == DELIM) + if (*p == DELIM) { bufsz++; /* number of DELIM plus one */ + } } - bufsz *= wcslen(pythonhome); + bufsz *= wcslen(calculate->home); } - else - bufsz = 0; bufsz += wcslen(PYTHONPATH) + 1; - bufsz += wcslen(argv0_path) + 1; - if (userpath) - bufsz += wcslen(userpath) + 1; - if (machinepath) - bufsz += wcslen(machinepath) + 1; - bufsz += wcslen(zip_path) + 1; - if (envpath != NULL) - bufsz += wcslen(envpath) + 1; - - module_search_path = buf = PyMem_RawMalloc(bufsz*sizeof(wchar_t)); + bufsz += wcslen(calculate->argv0_path) + 1; + if (calculate->user_path) { + bufsz += wcslen(calculate->user_path) + 1; + } + if (calculate->machine_path) { + bufsz += wcslen(calculate->machine_path) + 1; + } + bufsz += wcslen(calculate->zip_path) + 1; + if (calculate->module_search_path_env != NULL) { + bufsz += wcslen(calculate->module_search_path_env) + 1; + } + + wchar_t *buf, *start_buf; + buf = PyMem_RawMalloc(bufsz * sizeof(wchar_t)); if (buf == NULL) { /* We can't exit, so print a warning and limp along */ fprintf(stderr, "Can't malloc dynamic PYTHONPATH.\n"); - if (envpath) { + if (calculate->module_search_path_env) { fprintf(stderr, "Using environment $PYTHONPATH.\n"); - module_search_path = envpath; + config->module_search_path = calculate->module_search_path_env; } else { fprintf(stderr, "Using default static path.\n"); - module_search_path = PYTHONPATH; + config->module_search_path = PYTHONPATH; } - PyMem_RawFree(machinepath); - PyMem_RawFree(userpath); return; } + start_buf = buf; - if (envpath) { - if (wcscpy_s(buf, bufsz - (buf - module_search_path), envpath)) + if (calculate->module_search_path_env) { + if (wcscpy_s(buf, bufsz - (buf - start_buf), calculate->module_search_path_env)) { Py_FatalError("buffer overflow in getpathp.c's calculate_path()"); + } buf = wcschr(buf, L'\0'); *buf++ = DELIM; } - if (zip_path[0]) { - if (wcscpy_s(buf, bufsz - (buf - module_search_path), zip_path)) + if (calculate->zip_path[0]) { + if (wcscpy_s(buf, bufsz - (buf - start_buf), calculate->zip_path)) { Py_FatalError("buffer overflow in getpathp.c's calculate_path()"); + } buf = wcschr(buf, L'\0'); *buf++ = DELIM; } - if (userpath) { - if (wcscpy_s(buf, bufsz - (buf - module_search_path), userpath)) + if (calculate->user_path) { + if (wcscpy_s(buf, bufsz - (buf - start_buf), calculate->user_path)) { Py_FatalError("buffer overflow in getpathp.c's calculate_path()"); + } buf = wcschr(buf, L'\0'); *buf++ = DELIM; - PyMem_RawFree(userpath); } - if (machinepath) { - if (wcscpy_s(buf, bufsz - (buf - module_search_path), machinepath)) + if (calculate->machine_path) { + if (wcscpy_s(buf, bufsz - (buf - start_buf), calculate->machine_path)) { Py_FatalError("buffer overflow in getpathp.c's calculate_path()"); + } buf = wcschr(buf, L'\0'); *buf++ = DELIM; - PyMem_RawFree(machinepath); } - if (pythonhome == NULL) { + if (calculate->home == NULL) { if (!skipdefault) { - if (wcscpy_s(buf, bufsz - (buf - module_search_path), PYTHONPATH)) + if (wcscpy_s(buf, bufsz - (buf - start_buf), PYTHONPATH)) { Py_FatalError("buffer overflow in getpathp.c's calculate_path()"); + } buf = wcschr(buf, L'\0'); *buf++ = DELIM; } @@ -816,13 +940,16 @@ calculate_path(const _PyMainInterpreterConfig *config) size_t n; for (;;) { q = wcschr(p, DELIM); - if (q == NULL) + if (q == NULL) { n = wcslen(p); - else + } + else { n = q-p; + } if (p[0] == '.' && is_sep(p[1])) { - if (wcscpy_s(buf, bufsz - (buf - module_search_path), pythonhome)) + if (wcscpy_s(buf, bufsz - (buf - start_buf), calculate->home)) { Py_FatalError("buffer overflow in getpathp.c's calculate_path()"); + } buf = wcschr(buf, L'\0'); p++; n--; @@ -830,17 +957,19 @@ calculate_path(const _PyMainInterpreterConfig *config) wcsncpy(buf, p, n); buf += n; *buf++ = DELIM; - if (q == NULL) + if (q == NULL) { break; + } p = q+1; } } - if (argv0_path) { - wcscpy(buf, argv0_path); + if (calculate->argv0_path) { + wcscpy(buf, calculate->argv0_path); buf = wcschr(buf, L'\0'); *buf++ = DELIM; } *(buf - 1) = L'\0'; + /* Now to pull one last hack/trick. If sys.prefix is empty, then try and find it somewhere on the paths we calculated. We scan backwards, as our general policy @@ -849,7 +978,7 @@ calculate_path(const _PyMainInterpreterConfig *config) on the path, and that our 'prefix' directory is the parent of that. */ - if (*prefix==L'\0') { + if (config->prefix[0] == L'\0') { wchar_t lookBuf[MAXPATHLEN+1]; wchar_t *look = buf - 1; /* 'buf' is at the end of the buffer */ while (1) { @@ -859,84 +988,129 @@ calculate_path(const _PyMainInterpreterConfig *config) start of the path in question - even if this is one character before the start of the buffer */ - while (look >= module_search_path && *look != DELIM) + while (look >= start_buf && *look != DELIM) look--; nchars = lookEnd-look; wcsncpy(lookBuf, look+1, nchars); lookBuf[nchars] = L'\0'; /* Up one level to the parent */ reduce(lookBuf); - if (search_for_prefix(lookBuf, LANDMARK)) { + if (search_for_prefix(config->prefix, lookBuf, LANDMARK)) { break; } /* If we are out of paths to search - give up */ - if (look < module_search_path) + if (look < start_buf) { break; + } look--; } } + + config->module_search_path = start_buf; +} + + +static void +calculate_free(PyCalculatePath *calculate) +{ + PyMem_RawFree(calculate->machine_path); + PyMem_RawFree(calculate->user_path); } +static void +calculate_path(const _PyMainInterpreterConfig *main_config) +{ + PyCalculatePath calculate; + memset(&calculate, 0, sizeof(calculate)); + + _PyInitError err = calculate_init(&calculate, main_config); + if (_Py_INIT_FAILED(err)) { + calculate_free(&calculate); + _Py_FatalInitError(err); + } + + PyPathConfig new_path_config; + memset(&new_path_config, 0, sizeof(new_path_config)); + + calculate_path_impl(&calculate, &new_path_config, main_config); + path_config = new_path_config; + + calculate_free(&calculate); +} + + /* External interface */ void Py_SetPath(const wchar_t *path) { - if (module_search_path != NULL) { - PyMem_RawFree(module_search_path); - module_search_path = NULL; - } - if (path != NULL) { - extern wchar_t *Py_GetProgramName(void); - wchar_t *prog = Py_GetProgramName(); - wcsncpy(progpath, prog, MAXPATHLEN); - prefix[0] = L'\0'; - module_search_path = PyMem_RawMalloc((wcslen(path) + 1) * sizeof(wchar_t)); - if (module_search_path != NULL) - wcscpy(module_search_path, path); + if (path_config.module_search_path != NULL) { + PyMem_RawFree(path_config.module_search_path); + path_config.module_search_path = NULL; + } + + if (path == NULL) { + return; + } + + wchar_t *prog = Py_GetProgramName(); + wcsncpy(path_config.progpath, prog, MAXPATHLEN); + path_config.prefix[0] = L'\0'; + path_config.module_search_path = PyMem_RawMalloc((wcslen(path) + 1) * sizeof(wchar_t)); + if (path_config.module_search_path != NULL) { + wcscpy(path_config.module_search_path, path); } } + wchar_t * -_Py_GetPathWithConfig(const _PyMainInterpreterConfig *config) +_Py_GetPathWithConfig(const _PyMainInterpreterConfig *main_config) { - if (!module_search_path) { - calculate_path(config); + if (!path_config.module_search_path) { + calculate_path(main_config); } - return module_search_path; + return path_config.module_search_path; } + wchar_t * Py_GetPath(void) { - if (!module_search_path) + if (!path_config.module_search_path) { calculate_path(NULL); - return module_search_path; + } + return path_config.module_search_path; } + wchar_t * Py_GetPrefix(void) { - if (!module_search_path) + if (!path_config.module_search_path) { calculate_path(NULL); - return prefix; + } + return path_config.prefix; } + wchar_t * Py_GetExecPrefix(void) { return Py_GetPrefix(); } + wchar_t * Py_GetProgramFullPath(void) { - if (!module_search_path) + if (!path_config.module_search_path) { calculate_path(NULL); - return progpath; + } + return path_config.progpath; } + /* Load python3.dll before loading any extension module that might refer to it. That way, we can be sure that always the python3.dll corresponding to this python DLL is loaded, not a python3.dll that might be on the path @@ -950,20 +1124,23 @@ _Py_CheckPython3() { wchar_t py3path[MAXPATHLEN+1]; wchar_t *s; - if (python3_checked) + if (python3_checked) { return hPython3 != NULL; + } python3_checked = 1; /* If there is a python3.dll next to the python3y.dll, assume this is a build tree; use that DLL */ - wcscpy(py3path, dllpath); + wcscpy(py3path, path_config.dllpath); s = wcsrchr(py3path, L'\\'); - if (!s) + if (!s) { s = py3path; + } wcscpy(s, L"\\python3.dll"); hPython3 = LoadLibraryExW(py3path, NULL, LOAD_WITH_ALTERED_SEARCH_PATH); - if (hPython3 != NULL) + if (hPython3 != NULL) { return 1; + } /* Check sys.prefix\DLLs\python3.dll */ wcscpy(py3path, Py_GetPrefix()); diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 5bbbbc68f08ecc..8d2ec4e91c3891 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1477,8 +1477,9 @@ Py_SetPythonHome(wchar_t *home) default_home = home; } -wchar_t * -_Py_GetPythonHomeWithConfig(const _PyMainInterpreterConfig *config) + +_PyInitError +_Py_GetPythonHomeWithConfig(const _PyMainInterpreterConfig *config, wchar_t **homep) { /* Use a static buffer to avoid heap memory allocation failure. Py_GetPythonHome() doesn't allow to report error, and the caller @@ -1486,32 +1487,40 @@ _Py_GetPythonHomeWithConfig(const _PyMainInterpreterConfig *config) static wchar_t buffer[MAXPATHLEN+1]; if (default_home) { - return default_home; + *homep = default_home; + return _Py_INIT_OK(); } if (config) { - return config->pythonhome; + *homep = config->pythonhome; + return _Py_INIT_OK(); } char *home = Py_GETENV("PYTHONHOME"); if (!home) { - return NULL; + *homep = NULL; + return _Py_INIT_OK(); } size_t size = Py_ARRAY_LENGTH(buffer); size_t r = mbstowcs(buffer, home, size); if (r == (size_t)-1 || r >= size) { /* conversion failed or the static buffer is too small */ - return NULL; + *homep = NULL; + return _Py_INIT_ERR("failed to decode PYTHONHOME environment variable"); } - return buffer; + *homep = buffer; + return _Py_INIT_OK(); } wchar_t * Py_GetPythonHome(void) { - return _Py_GetPythonHomeWithConfig(NULL); + wchar_t *home; + /* Ignore error */ + (void)_Py_GetPythonHomeWithConfig(NULL, &home); + return home; } /* Add the __main__ module */ From b98f1715a35d2cbd1e9e45e1e7ae51a39e00dc4a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 23 Nov 2017 17:13:44 +0100 Subject: [PATCH 20/75] bpo-27535: Cleanup create_filter() (#4516) create_filter() now expects the action as a _Py_Identifier which avoids string comparison, and more important, to avoid handling the "unknown action" annoying case. --- Python/_warnings.c | 56 +++++++++++++++------------------------------- 1 file changed, 18 insertions(+), 38 deletions(-) diff --git a/Python/_warnings.c b/Python/_warnings.c index 36d649fda10c54..086a70d7e68f0a 100644 --- a/Python/_warnings.c +++ b/Python/_warnings.c @@ -11,6 +11,10 @@ MODULE_NAME " provides basic warning filtering support.\n" _Py_IDENTIFIER(argv); _Py_IDENTIFIER(stderr); +_Py_IDENTIFIER(ignore); +_Py_IDENTIFIER(error); +_Py_IDENTIFIER(always); +_Py_static_string(PyId_default, "default"); static int check_matched(PyObject *obj, PyObject *arg) @@ -1149,31 +1153,8 @@ static PyMethodDef warnings_functions[] = { static PyObject * -create_filter(PyObject *category, const char *action) +create_filter(PyObject *category, _Py_Identifier *id) { - _Py_IDENTIFIER(ignore); - _Py_IDENTIFIER(error); - _Py_IDENTIFIER(always); - _Py_static_string(PyId_default, "default"); - _Py_Identifier *id; - - if (!strcmp(action, "ignore")) { - id = &PyId_ignore; - } - else if (!strcmp(action, "error")) { - id = &PyId_error; - } - else if (!strcmp(action, "default")) { - id = &PyId_default; - } - else if (!strcmp(action, "always")) { - id = &PyId_always; - } - else { - PyErr_SetString(PyExc_ValueError, "unknown action"); - return NULL; - } - PyObject *action_str = _PyUnicode_FromId(id); if (action_str == NULL) { return NULL; @@ -1199,48 +1180,47 @@ init_filters(const _PyCoreConfig *config) } #endif PyObject *filters = PyList_New(count); - unsigned int pos = 0; /* Post-incremented in each use. */ - unsigned int x; - const char *bytes_action, *resource_action; - if (filters == NULL) return NULL; + size_t pos = 0; /* Post-incremented in each use. */ #ifndef Py_DEBUG if (!dev_mode) { PyList_SET_ITEM(filters, pos++, - create_filter(PyExc_DeprecationWarning, "ignore")); + create_filter(PyExc_DeprecationWarning, &PyId_ignore)); PyList_SET_ITEM(filters, pos++, - create_filter(PyExc_PendingDeprecationWarning, "ignore")); + create_filter(PyExc_PendingDeprecationWarning, &PyId_ignore)); PyList_SET_ITEM(filters, pos++, - create_filter(PyExc_ImportWarning, "ignore")); + create_filter(PyExc_ImportWarning, &PyId_ignore)); } #endif + _Py_Identifier *bytes_action; if (Py_BytesWarningFlag > 1) - bytes_action = "error"; + bytes_action = &PyId_error; else if (Py_BytesWarningFlag) - bytes_action = "default"; + bytes_action = &PyId_default; else - bytes_action = "ignore"; + bytes_action = &PyId_ignore; PyList_SET_ITEM(filters, pos++, create_filter(PyExc_BytesWarning, bytes_action)); + _Py_Identifier *resource_action; /* resource usage warnings are enabled by default in pydebug mode */ #ifdef Py_DEBUG - resource_action = "always"; + resource_action = &PyId_always; #else - resource_action = (dev_mode ? "always" : "ignore"); + resource_action = (dev_mode ? &PyId_always : &PyId_ignore); #endif PyList_SET_ITEM(filters, pos++, create_filter(PyExc_ResourceWarning, resource_action)); if (dev_mode) { PyList_SET_ITEM(filters, pos++, - create_filter(PyExc_Warning, "default")); + create_filter(PyExc_Warning, &PyId_default)); } - for (x = 0; x < pos; x += 1) { + for (size_t x = 0; x < pos; x++) { if (PyList_GET_ITEM(filters, x) == NULL) { Py_DECREF(filters); return NULL; From b9197959c186f9061bd74d0adc20e96556426db4 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 23 Nov 2017 19:02:04 +0100 Subject: [PATCH 21/75] bpo-32030: Fix calculate_path() on macOS (#4526) --- Modules/getpath.c | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/Modules/getpath.c b/Modules/getpath.c index d8125ae2cac8eb..48bdd0f21daca7 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -589,6 +589,15 @@ calculate_reduce_exec_prefix(PyCalculatePath *calculate, PyPathConfig *config) static void calculate_progpath(PyCalculatePath *calculate, PyPathConfig *config) { +#ifdef __APPLE__ +#if MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_4 + uint32_t nsexeclength = MAXPATHLEN; +#else + unsigned long nsexeclength = MAXPATHLEN; +#endif + char execpath[MAXPATHLEN+1]; +#endif + /* If there is no slash in the argv0 path, then we have to * assume python is on the user's $PATH, since there's no * other way to find a directory to start the search from. If @@ -597,15 +606,7 @@ calculate_progpath(PyCalculatePath *calculate, PyPathConfig *config) if (wcschr(calculate->prog, SEP)) { wcsncpy(config->progpath, calculate->prog, MAXPATHLEN); } - #ifdef __APPLE__ -#if MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_4 - uint32_t nsexeclength = MAXPATHLEN; -#else - unsigned long nsexeclength = MAXPATHLEN; -#endif - char execpath[MAXPATHLEN+1]; - /* On Mac OS X, if a script uses an interpreter of the form * "#!/opt/python2.3/bin/python", the kernel only passes "python" * as argv[0], which falls through to the $PATH search below. From 6a54c676e63517653d3d4a1e164bdd0fd45132d8 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 23 Nov 2017 19:02:23 +0100 Subject: [PATCH 22/75] bpo-31979: Remove unused align_maxchar() function (#4527) --- Objects/unicodeobject.c | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index f6b2b65d1ffba5..8d4fea8ede15da 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -2171,19 +2171,6 @@ kind_maxchar_limit(unsigned int kind) } } -static inline Py_UCS4 -align_maxchar(Py_UCS4 maxchar) -{ - if (maxchar <= 127) - return 127; - else if (maxchar <= 255) - return 255; - else if (maxchar <= 65535) - return 65535; - else - return MAX_UNICODE; -} - static PyObject* _PyUnicode_FromUCS1(const Py_UCS1* u, Py_ssize_t size) { From dcaed6b2d954786eb5369ec2e8dfdeefe3cdc6ae Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Thu, 23 Nov 2017 21:34:20 +0300 Subject: [PATCH 23/75] bpo-19610: setup() now raises TypeError for invalid types (GH-4519) The Distribution class now explicitly raises an exception when 'classifiers', 'keywords' and 'platforms' fields are not specified as a list. --- Doc/distutils/apiref.rst | 4 ++ Doc/distutils/setupscript.rst | 39 +++++++++++----- Doc/whatsnew/3.7.rst | 6 +++ Lib/distutils/dist.py | 26 +++++++++++ Lib/distutils/tests/test_dist.py | 44 +++++++++++++++++++ .../2017-11-23-16-15-55.bpo-19610.Dlca2P.rst | 5 +++ 6 files changed, 112 insertions(+), 12 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2017-11-23-16-15-55.bpo-19610.Dlca2P.rst diff --git a/Doc/distutils/apiref.rst b/Doc/distutils/apiref.rst index 7cde1a0701eb8f..ced8837d37363f 100644 --- a/Doc/distutils/apiref.rst +++ b/Doc/distutils/apiref.rst @@ -285,6 +285,10 @@ the full reference. See the :func:`setup` function for a list of keyword arguments accepted by the Distribution constructor. :func:`setup` creates a Distribution instance. + .. versionchanged:: 3.7 + :class:`~distutils.core.Distribution` now raises a :exc:`TypeError` if + ``classifiers``, ``keywords`` and ``platforms`` fields are not specified + as a list. .. class:: Command diff --git a/Doc/distutils/setupscript.rst b/Doc/distutils/setupscript.rst index 38e0202e4ac1f6..542ad544843fde 100644 --- a/Doc/distutils/setupscript.rst +++ b/Doc/distutils/setupscript.rst @@ -581,17 +581,19 @@ This information includes: | | description of the | | | | | package | | | +----------------------+---------------------------+-----------------+--------+ -| ``long_description`` | longer description of the | long string | \(5) | +| ``long_description`` | longer description of the | long string | \(4) | | | package | | | +----------------------+---------------------------+-----------------+--------+ -| ``download_url`` | location where the | URL | \(4) | +| ``download_url`` | location where the | URL | | | | package may be downloaded | | | +----------------------+---------------------------+-----------------+--------+ -| ``classifiers`` | a list of classifiers | list of strings | \(4) | +| ``classifiers`` | a list of classifiers | list of strings | (6)(7) | +----------------------+---------------------------+-----------------+--------+ -| ``platforms`` | a list of platforms | list of strings | | +| ``platforms`` | a list of platforms | list of strings | (6)(8) | +----------------------+---------------------------+-----------------+--------+ -| ``license`` | license for the package | short string | \(6) | +| ``keywords`` | a list of keywords | list of strings | (6)(8) | ++----------------------+---------------------------+-----------------+--------+ +| ``license`` | license for the package | short string | \(5) | +----------------------+---------------------------+-----------------+--------+ Notes: @@ -607,22 +609,30 @@ Notes: provided, distutils lists it as the author in :file:`PKG-INFO`. (4) - These fields should not be used if your package is to be compatible with Python - versions prior to 2.2.3 or 2.3. The list is available from the `PyPI website - `_. - -(5) The ``long_description`` field is used by PyPI when you are :ref:`registering ` a package, to :ref:`build its home page `. -(6) +(5) The ``license`` field is a text indicating the license covering the package where the license is not a selection from the "License" Trove classifiers. See the ``Classifier`` field. Notice that there's a ``licence`` distribution option which is deprecated but still acts as an alias for ``license``. +(6) + This field must be a list. + +(7) + The valid classifiers are listed on + `PyPI `_. + +(8) + To preserve backward compatibility, this field also accepts a string. If + you pass a comma-separated string ``'foo, bar'``, it will be converted to + ``['foo', 'bar']``, Otherwise, it will be converted to a list of one + string. + 'short string' A single line of text, not more than 200 characters. @@ -650,7 +660,7 @@ information is sometimes used to indicate sub-releases. These are 1.0.1a2 the second alpha release of the first patch version of 1.0 -``classifiers`` are specified in a Python list:: +``classifiers`` must be specified in a list:: setup(..., classifiers=[ @@ -671,6 +681,11 @@ information is sometimes used to indicate sub-releases. These are ], ) +.. versionchanged:: 3.7 + :class:`~distutils.core.setup` now raises a :exc:`TypeError` if + ``classifiers``, ``keywords`` and ``platforms`` fields are not specified + as a list. + .. _debug-setup-script: Debugging the setup script diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index 71e8358a420b61..514c3c293c080c 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -298,6 +298,12 @@ README.rst is now included in the list of distutils standard READMEs and therefore included in source distributions. (Contributed by Ryan Gonzalez in :issue:`11913`.) +:class:`distutils.core.setup` now raises a :exc:`TypeError` if +``classifiers``, ``keywords`` and ``platforms`` fields are not specified +as a list. However, to minimize backwards incompatibility concerns, +``keywords`` and ``platforms`` fields still accept a comma separated string. +(Contributed by Berker Peksag in :issue:`19610`.) + http.client ----------- diff --git a/Lib/distutils/dist.py b/Lib/distutils/dist.py index 62a24516cfafe5..78c29ede6c2c4e 100644 --- a/Lib/distutils/dist.py +++ b/Lib/distutils/dist.py @@ -1188,12 +1188,38 @@ def get_long_description(self): def get_keywords(self): return self.keywords or [] + def set_keywords(self, value): + # If 'keywords' is a string, it will be converted to a list + # by Distribution.finalize_options(). To maintain backwards + # compatibility, do not raise an exception if 'keywords' is + # a string. + if not isinstance(value, (list, str)): + msg = "'keywords' should be a 'list', not %r" + raise TypeError(msg % type(value).__name__) + self.keywords = value + def get_platforms(self): return self.platforms or ["UNKNOWN"] + def set_platforms(self, value): + # If 'platforms' is a string, it will be converted to a list + # by Distribution.finalize_options(). To maintain backwards + # compatibility, do not raise an exception if 'platforms' is + # a string. + if not isinstance(value, (list, str)): + msg = "'platforms' should be a 'list', not %r" + raise TypeError(msg % type(value).__name__) + self.platforms = value + def get_classifiers(self): return self.classifiers or [] + def set_classifiers(self, value): + if not isinstance(value, list): + msg = "'classifiers' should be a 'list', not %r" + raise TypeError(msg % type(value).__name__) + self.classifiers = value + def get_download_url(self): return self.download_url or "UNKNOWN" diff --git a/Lib/distutils/tests/test_dist.py b/Lib/distutils/tests/test_dist.py index 1f104cef675d9a..50b456ec9471c7 100644 --- a/Lib/distutils/tests/test_dist.py +++ b/Lib/distutils/tests/test_dist.py @@ -195,6 +195,13 @@ def test_finalize_options(self): self.assertEqual(dist.metadata.platforms, ['one', 'two']) self.assertEqual(dist.metadata.keywords, ['one', 'two']) + attrs = {'keywords': 'foo bar', + 'platforms': 'foo bar'} + dist = Distribution(attrs=attrs) + dist.finalize_options() + self.assertEqual(dist.metadata.platforms, ['foo bar']) + self.assertEqual(dist.metadata.keywords, ['foo bar']) + def test_get_command_packages(self): dist = Distribution() self.assertEqual(dist.command_packages, None) @@ -338,9 +345,46 @@ def test_classifier(self): attrs = {'name': 'Boa', 'version': '3.0', 'classifiers': ['Programming Language :: Python :: 3']} dist = Distribution(attrs) + self.assertEqual(dist.get_classifiers(), + ['Programming Language :: Python :: 3']) meta = self.format_metadata(dist) self.assertIn('Metadata-Version: 1.1', meta) + def test_classifier_invalid_type(self): + attrs = {'name': 'Boa', 'version': '3.0', + 'classifiers': ('Programming Language :: Python :: 3',)} + msg = "'classifiers' should be a 'list', not 'tuple'" + with self.assertRaises(TypeError, msg=msg): + Distribution(attrs) + + def test_keywords(self): + attrs = {'name': 'Monty', 'version': '1.0', + 'keywords': ['spam', 'eggs', 'life of brian']} + dist = Distribution(attrs) + self.assertEqual(dist.get_keywords(), + ['spam', 'eggs', 'life of brian']) + + def test_keywords_invalid_type(self): + attrs = {'name': 'Monty', 'version': '1.0', + 'keywords': ('spam', 'eggs', 'life of brian')} + msg = "'keywords' should be a 'list', not 'tuple'" + with self.assertRaises(TypeError, msg=msg): + Distribution(attrs) + + def test_platforms(self): + attrs = {'name': 'Monty', 'version': '1.0', + 'platforms': ['GNU/Linux', 'Some Evil Platform']} + dist = Distribution(attrs) + self.assertEqual(dist.get_platforms(), + ['GNU/Linux', 'Some Evil Platform']) + + def test_platforms_invalid_types(self): + attrs = {'name': 'Monty', 'version': '1.0', + 'platforms': ('GNU/Linux', 'Some Evil Platform')} + msg = "'platforms' should be a 'list', not 'tuple'" + with self.assertRaises(TypeError, msg=msg): + Distribution(attrs) + def test_download_url(self): attrs = {'name': 'Boa', 'version': '3.0', 'download_url': 'http://example.org/boa'} diff --git a/Misc/NEWS.d/next/Library/2017-11-23-16-15-55.bpo-19610.Dlca2P.rst b/Misc/NEWS.d/next/Library/2017-11-23-16-15-55.bpo-19610.Dlca2P.rst new file mode 100644 index 00000000000000..5ea87a45722c0e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-11-23-16-15-55.bpo-19610.Dlca2P.rst @@ -0,0 +1,5 @@ +``setup()`` now raises :exc:`TypeError` for invalid types. + +The ``distutils.dist.Distribution`` class now explicitly raises an exception +when ``classifiers``, ``keywords`` and ``platforms`` fields are not +specified as a list. From 0858495a50e19defde786a4ec050ec182e920f46 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Thu, 23 Nov 2017 13:32:23 -0800 Subject: [PATCH 24/75] bpo-32099 Add deque variant of roundrobin() recipe (#4497) * Minor wording tweaks --- Doc/library/collections.rst | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst index cda829694a3277..4b0d8c048ae7d9 100644 --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -618,6 +618,25 @@ added elements by appending to the right and popping to the left:: d.append(elem) yield s / n +A `round-robin scheduler +`_ can be implemented with +input iterators stored in a :class:`deque`. Values are yielded from the active +iterator in position zero. If that iterator is exhausted, it can be removed +with :meth:`~deque.popleft`; otherwise, it can be cycled back to the end with +the :meth:`~deque.rotate` method:: + + def roundrobin(*iterables): + "roundrobin('ABC', 'D', 'EF') --> A D E B F C" + iterators = deque(map(iter, iterables)) + while iterators: + try: + while True: + yield next(iterators[0]) + iterators.rotate(-1) + except StopIteration: + # Remove an exhausted iterator. + iterators.popleft() + The :meth:`rotate` method provides a way to implement :class:`deque` slicing and deletion. For example, a pure Python implementation of ``del d[n]`` relies on the :meth:`rotate` method to position elements to be popped:: From 3df02dbc8e197053105f9dffeae40b04ec66766e Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Fri, 24 Nov 2017 02:40:26 +0300 Subject: [PATCH 25/75] bpo-31325: Fix usage of namedtuple in RobotFileParser.parse() (#4529) --- Doc/library/urllib.robotparser.rst | 8 ++++---- Lib/test/test_robotparser.py | 9 ++++++--- Lib/urllib/robotparser.py | 9 ++++----- .../Library/2017-11-23-22-12-11.bpo-31325.8jAUxN.rst | 5 +++++ 4 files changed, 19 insertions(+), 12 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2017-11-23-22-12-11.bpo-31325.8jAUxN.rst diff --git a/Doc/library/urllib.robotparser.rst b/Doc/library/urllib.robotparser.rst index 7d31932f9656e4..e3b90e673caaf0 100644 --- a/Doc/library/urllib.robotparser.rst +++ b/Doc/library/urllib.robotparser.rst @@ -69,10 +69,10 @@ structure of :file:`robots.txt` files, see http://www.robotstxt.org/orig.html. .. method:: request_rate(useragent) Returns the contents of the ``Request-rate`` parameter from - ``robots.txt`` in the form of a :func:`~collections.namedtuple` - ``(requests, seconds)``. If there is no such parameter or it doesn't - apply to the *useragent* specified or the ``robots.txt`` entry for this - parameter has invalid syntax, return ``None``. + ``robots.txt`` as a :term:`named tuple` ``RequestRate(requests, seconds)``. + If there is no such parameter or it doesn't apply to the *useragent* + specified or the ``robots.txt`` entry for this parameter has invalid + syntax, return ``None``. .. versionadded:: 3.6 diff --git a/Lib/test/test_robotparser.py b/Lib/test/test_robotparser.py index 5c1a571f1b6d70..75198b70ad4ff5 100644 --- a/Lib/test/test_robotparser.py +++ b/Lib/test/test_robotparser.py @@ -3,7 +3,6 @@ import threading import unittest import urllib.robotparser -from collections import namedtuple from test import support from http.server import BaseHTTPRequestHandler, HTTPServer @@ -87,6 +86,10 @@ def test_request_rate(self): self.parser.crawl_delay(agent), self.crawl_delay ) if self.request_rate: + self.assertIsInstance( + self.parser.request_rate(agent), + urllib.robotparser.RequestRate + ) self.assertEqual( self.parser.request_rate(agent).requests, self.request_rate.requests @@ -108,7 +111,7 @@ class CrawlDelayAndRequestRateTest(BaseRequestRateTest, unittest.TestCase): Disallow: /%7ejoe/index.html """ agent = 'figtree' - request_rate = namedtuple('req_rate', 'requests seconds')(9, 30) + request_rate = urllib.robotparser.RequestRate(9, 30) crawl_delay = 3 good = [('figtree', '/foo.html')] bad = ['/tmp', '/tmp.html', '/tmp/a.html', '/a%3cd.html', '/a%3Cd.html', @@ -237,7 +240,7 @@ class DefaultEntryTest(BaseRequestRateTest, unittest.TestCase): Request-rate: 3/15 Disallow: /cyberworld/map/ """ - request_rate = namedtuple('req_rate', 'requests seconds')(3, 15) + request_rate = urllib.robotparser.RequestRate(3, 15) crawl_delay = 1 good = ['/', '/test.html'] bad = ['/cyberworld/map/index.html'] diff --git a/Lib/urllib/robotparser.py b/Lib/urllib/robotparser.py index 9dab4c1c3a8880..daac29c68dc36d 100644 --- a/Lib/urllib/robotparser.py +++ b/Lib/urllib/robotparser.py @@ -16,6 +16,9 @@ __all__ = ["RobotFileParser"] +RequestRate = collections.namedtuple("RequestRate", "requests seconds") + + class RobotFileParser: """ This class provides a set of methods to read, parse and answer questions about a single robots.txt file. @@ -136,11 +139,7 @@ def parse(self, lines): # check if all values are sane if (len(numbers) == 2 and numbers[0].strip().isdigit() and numbers[1].strip().isdigit()): - req_rate = collections.namedtuple('req_rate', - 'requests seconds') - entry.req_rate = req_rate - entry.req_rate.requests = int(numbers[0]) - entry.req_rate.seconds = int(numbers[1]) + entry.req_rate = RequestRate(int(numbers[0]), int(numbers[1])) state = 2 if state == 2: self._add_entry(entry) diff --git a/Misc/NEWS.d/next/Library/2017-11-23-22-12-11.bpo-31325.8jAUxN.rst b/Misc/NEWS.d/next/Library/2017-11-23-22-12-11.bpo-31325.8jAUxN.rst new file mode 100644 index 00000000000000..89a193c9ef5b8b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-11-23-22-12-11.bpo-31325.8jAUxN.rst @@ -0,0 +1,5 @@ +Fix wrong usage of :func:`collections.namedtuple` in +the :meth:`RobotFileParser.parse() ` +method. + +Initial patch by Robin Wellner. From cdfe910e746e1d0fc43429b8cc3384a65a19b358 Mon Sep 17 00:00:00 2001 From: Emanuele Gaifas Date: Fri, 24 Nov 2017 09:49:57 +0100 Subject: [PATCH 26/75] Extending Python Doc minor updates (GH-4518) Move footnote markers to be closer to the related terminology: before the end of the sentence, instead of after. --- Doc/extending/extending.rst | 6 +++--- Doc/extending/newtypes.rst | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Doc/extending/extending.rst b/Doc/extending/extending.rst index 7c273533aba5b5..ea1c29a397ed86 100644 --- a/Doc/extending/extending.rst +++ b/Doc/extending/extending.rst @@ -40,7 +40,7 @@ A Simple Example Let's create an extension module called ``spam`` (the favorite food of Monty Python fans...) and let's say we want to create a Python interface to the C -library function :c:func:`system`. [#]_ This function takes a null-terminated +library function :c:func:`system` [#]_. This function takes a null-terminated character string as argument and returns an integer. We want this function to be callable from Python as follows:: @@ -917,7 +917,7 @@ It is also possible to :dfn:`borrow` [#]_ a reference to an object. The borrower of a reference should not call :c:func:`Py_DECREF`. The borrower must not hold on to the object longer than the owner from which it was borrowed. Using a borrowed reference after the owner has disposed of it risks using freed -memory and should be avoided completely. [#]_ +memory and should be avoided completely [#]_. The advantage of borrowing over owning a reference is that you don't need to take care of disposing of the reference on all possible paths through the code @@ -1088,7 +1088,7 @@ checking. The C function calling mechanism guarantees that the argument list passed to C functions (``args`` in the examples) is never *NULL* --- in fact it guarantees -that it is always a tuple. [#]_ +that it is always a tuple [#]_. It is a severe error to ever let a *NULL* pointer "escape" to the Python user. diff --git a/Doc/extending/newtypes.rst b/Doc/extending/newtypes.rst index 0e36ba0aec07ae..62fbdb87a53000 100644 --- a/Doc/extending/newtypes.rst +++ b/Doc/extending/newtypes.rst @@ -659,7 +659,7 @@ Fortunately, Python's cyclic-garbage collector will eventually figure out that the list is garbage and free it. In the second version of the :class:`Noddy` example, we allowed any kind of -object to be stored in the :attr:`first` or :attr:`last` attributes. [#]_ This +object to be stored in the :attr:`first` or :attr:`last` attributes [#]_. This means that :class:`Noddy` objects can participate in cycles:: >>> import noddy2 From 4864a619dc1cc9092780ccf5a6327e8abf66133d Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Fri, 24 Nov 2017 12:53:58 +0300 Subject: [PATCH 27/75] bpo-12382: Make OpenDatabase() raise better exception messages (GH-4528) Previously, 'msilib.OpenDatabase()' function raised a cryptical exception message when it couldn't open or create an MSI file. For example: Traceback (most recent call last): File "", line 1, in _msi.MSIError: unknown error 6e --- Lib/test/test_msilib.py | 12 ++++++++++++ .../Library/2017-11-23-21-47-36.bpo-12382.xWT9k0.rst | 2 ++ PC/_msi.c | 6 ++++++ 3 files changed, 20 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2017-11-23-21-47-36.bpo-12382.xWT9k0.rst diff --git a/Lib/test/test_msilib.py b/Lib/test/test_msilib.py index 65ff3869c33cbe..4093134195a26e 100644 --- a/Lib/test/test_msilib.py +++ b/Lib/test/test_msilib.py @@ -1,4 +1,5 @@ """ Test suite for the code in msilib """ +import os.path import unittest from test.support import TESTFN, import_module, unlink msilib = import_module('msilib') @@ -41,6 +42,17 @@ def test_view_fetch_returns_none(self): ) self.addCleanup(unlink, db_path) + def test_database_open_failed(self): + with self.assertRaises(msilib.MSIError) as cm: + msilib.OpenDatabase('non-existent.msi', msilib.MSIDBOPEN_READONLY) + self.assertEqual(str(cm.exception), 'open failed') + + def test_database_create_failed(self): + db_path = os.path.join(TESTFN, 'test.msi') + with self.assertRaises(msilib.MSIError) as cm: + msilib.OpenDatabase(db_path, msilib.MSIDBOPEN_CREATE) + self.assertEqual(str(cm.exception), 'create failed') + class Test_make_id(unittest.TestCase): #http://msdn.microsoft.com/en-us/library/aa369212(v=vs.85).aspx diff --git a/Misc/NEWS.d/next/Library/2017-11-23-21-47-36.bpo-12382.xWT9k0.rst b/Misc/NEWS.d/next/Library/2017-11-23-21-47-36.bpo-12382.xWT9k0.rst new file mode 100644 index 00000000000000..d9b54259cc6ce3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-11-23-21-47-36.bpo-12382.xWT9k0.rst @@ -0,0 +1,2 @@ +:func:`msilib.OpenDatabase` now raises a better exception message when it +couldn't open or create an MSI file. Initial patch by William Tisäter. diff --git a/PC/_msi.c b/PC/_msi.c index a6a12e2010738b..8cc1fd59dd1926 100644 --- a/PC/_msi.c +++ b/PC/_msi.c @@ -315,6 +315,12 @@ msierror(int status) case ERROR_INVALID_PARAMETER: PyErr_SetString(MSIError, "invalid parameter"); return NULL; + case ERROR_OPEN_FAILED: + PyErr_SetString(MSIError, "open failed"); + return NULL; + case ERROR_CREATE_FAILED: + PyErr_SetString(MSIError, "create failed"); + return NULL; default: PyErr_Format(MSIError, "unknown error %x", status); return NULL; From 9e87e7776f7ace66baaf7247233afdabd00c2b44 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 24 Nov 2017 12:09:24 +0100 Subject: [PATCH 28/75] bpo-32096: Remove obj and mem from _PyRuntime (#4532) bpo-32096, bpo-30860: Partially revert the commit 2ebc5ce42a8a9e047e790aefbf9a94811569b2b6: * Move structures back from Include/internal/mem.h to Objects/obmalloc.c * Remove _PyObject_Initialize() and _PyMem_Initialize() * Remove Include/internal/pymalloc.h * Add test_capi.test_pre_initialization_api(): Make sure that it's possible to call Py_DecodeLocale(), and then call Py_SetProgramName() with the decoded string, before Py_Initialize(). PyMem_RawMalloc() and Py_DecodeLocale() can be called again before _PyRuntimeState_Init(). Co-Authored-By: Eric Snow --- Include/internal/mem.h | 48 - Include/internal/pymalloc.h | 443 ---------- Include/internal/pystate.h | 2 - Lib/test/test_capi.py | 10 + Makefile.pre.in | 1 - .../2017-11-24-01-13-58.bpo-32096.CQTHXJ.rst | 4 + Objects/obmalloc.c | 833 ++++++++++++++---- PCbuild/pythoncore.vcxproj | 1 - PCbuild/pythoncore.vcxproj.filters | 5 +- Parser/pgenmain.c | 2 - Programs/_testembed.c | 23 + Python/pystate.c | 2 - 12 files changed, 685 insertions(+), 689 deletions(-) delete mode 100644 Include/internal/pymalloc.h create mode 100644 Misc/NEWS.d/next/Core and Builtins/2017-11-24-01-13-58.bpo-32096.CQTHXJ.rst diff --git a/Include/internal/mem.h b/Include/internal/mem.h index 471cdf45df2766..a731e30e6af7d2 100644 --- a/Include/internal/mem.h +++ b/Include/internal/mem.h @@ -7,54 +7,6 @@ extern "C" { #include "objimpl.h" #include "pymem.h" -#ifdef WITH_PYMALLOC -#include "internal/pymalloc.h" -#endif - -/* Low-level memory runtime state */ - -struct _pymem_runtime_state { - struct _allocator_runtime_state { - PyMemAllocatorEx mem; - PyMemAllocatorEx obj; - PyMemAllocatorEx raw; - } allocators; -#ifdef WITH_PYMALLOC - /* Array of objects used to track chunks of memory (arenas). */ - struct arena_object* arenas; - /* The head of the singly-linked, NULL-terminated list of available - arena_objects. */ - struct arena_object* unused_arena_objects; - /* The head of the doubly-linked, NULL-terminated at each end, - list of arena_objects associated with arenas that have pools - available. */ - struct arena_object* usable_arenas; - /* Number of slots currently allocated in the `arenas` vector. */ - unsigned int maxarenas; - /* Number of arenas allocated that haven't been free()'d. */ - size_t narenas_currently_allocated; - /* High water mark (max value ever seen) for - * narenas_currently_allocated. */ - size_t narenas_highwater; - /* Total number of times malloc() called to allocate an arena. */ - size_t ntimes_arena_allocated; - poolp usedpools[MAX_POOLS]; - Py_ssize_t num_allocated_blocks; -#endif /* WITH_PYMALLOC */ - size_t serialno; /* incremented on each debug {m,re}alloc */ -}; - -PyAPI_FUNC(void) _PyMem_Initialize(struct _pymem_runtime_state *); - - -/* High-level memory runtime state */ - -struct _pyobj_runtime_state { - PyObjectArenaAllocator allocator_arenas; -}; - -PyAPI_FUNC(void) _PyObject_Initialize(struct _pyobj_runtime_state *); - /* GC runtime state */ diff --git a/Include/internal/pymalloc.h b/Include/internal/pymalloc.h deleted file mode 100644 index 723d9e7e671a8e..00000000000000 --- a/Include/internal/pymalloc.h +++ /dev/null @@ -1,443 +0,0 @@ - -/* An object allocator for Python. - - Here is an introduction to the layers of the Python memory architecture, - showing where the object allocator is actually used (layer +2), It is - called for every object allocation and deallocation (PyObject_New/Del), - unless the object-specific allocators implement a proprietary allocation - scheme (ex.: ints use a simple free list). This is also the place where - the cyclic garbage collector operates selectively on container objects. - - - Object-specific allocators - _____ ______ ______ ________ - [ int ] [ dict ] [ list ] ... [ string ] Python core | -+3 | <----- Object-specific memory -----> | <-- Non-object memory --> | - _______________________________ | | - [ Python's object allocator ] | | -+2 | ####### Object memory ####### | <------ Internal buffers ------> | - ______________________________________________________________ | - [ Python's raw memory allocator (PyMem_ API) ] | -+1 | <----- Python memory (under PyMem manager's control) ------> | | - __________________________________________________________________ - [ Underlying general-purpose allocator (ex: C library malloc) ] - 0 | <------ Virtual memory allocated for the python process -------> | - - ========================================================================= - _______________________________________________________________________ - [ OS-specific Virtual Memory Manager (VMM) ] --1 | <--- Kernel dynamic storage allocation & management (page-based) ---> | - __________________________________ __________________________________ - [ ] [ ] --2 | <-- Physical memory: ROM/RAM --> | | <-- Secondary storage (swap) --> | - -*/ -/*==========================================================================*/ - -/* A fast, special-purpose memory allocator for small blocks, to be used - on top of a general-purpose malloc -- heavily based on previous art. */ - -/* Vladimir Marangozov -- August 2000 */ - -/* - * "Memory management is where the rubber meets the road -- if we do the wrong - * thing at any level, the results will not be good. And if we don't make the - * levels work well together, we are in serious trouble." (1) - * - * (1) Paul R. Wilson, Mark S. Johnstone, Michael Neely, and David Boles, - * "Dynamic Storage Allocation: A Survey and Critical Review", - * in Proc. 1995 Int'l. Workshop on Memory Management, September 1995. - */ - -#ifndef Py_INTERNAL_PYMALLOC_H -#define Py_INTERNAL_PYMALLOC_H - -/* #undef WITH_MEMORY_LIMITS */ /* disable mem limit checks */ - -/*==========================================================================*/ - -/* - * Allocation strategy abstract: - * - * For small requests, the allocator sub-allocates blocks of memory. - * Requests greater than SMALL_REQUEST_THRESHOLD bytes are routed to the - * system's allocator. - * - * Small requests are grouped in size classes spaced 8 bytes apart, due - * to the required valid alignment of the returned address. Requests of - * a particular size are serviced from memory pools of 4K (one VMM page). - * Pools are fragmented on demand and contain free lists of blocks of one - * particular size class. In other words, there is a fixed-size allocator - * for each size class. Free pools are shared by the different allocators - * thus minimizing the space reserved for a particular size class. - * - * This allocation strategy is a variant of what is known as "simple - * segregated storage based on array of free lists". The main drawback of - * simple segregated storage is that we might end up with lot of reserved - * memory for the different free lists, which degenerate in time. To avoid - * this, we partition each free list in pools and we share dynamically the - * reserved space between all free lists. This technique is quite efficient - * for memory intensive programs which allocate mainly small-sized blocks. - * - * For small requests we have the following table: - * - * Request in bytes Size of allocated block Size class idx - * ---------------------------------------------------------------- - * 1-8 8 0 - * 9-16 16 1 - * 17-24 24 2 - * 25-32 32 3 - * 33-40 40 4 - * 41-48 48 5 - * 49-56 56 6 - * 57-64 64 7 - * 65-72 72 8 - * ... ... ... - * 497-504 504 62 - * 505-512 512 63 - * - * 0, SMALL_REQUEST_THRESHOLD + 1 and up: routed to the underlying - * allocator. - */ - -/*==========================================================================*/ - -/* - * -- Main tunable settings section -- - */ - -/* - * Alignment of addresses returned to the user. 8-bytes alignment works - * on most current architectures (with 32-bit or 64-bit address busses). - * The alignment value is also used for grouping small requests in size - * classes spaced ALIGNMENT bytes apart. - * - * You shouldn't change this unless you know what you are doing. - */ -#define ALIGNMENT 8 /* must be 2^N */ -#define ALIGNMENT_SHIFT 3 - -/* Return the number of bytes in size class I, as a uint. */ -#define INDEX2SIZE(I) (((unsigned int)(I) + 1) << ALIGNMENT_SHIFT) - -/* - * Max size threshold below which malloc requests are considered to be - * small enough in order to use preallocated memory pools. You can tune - * this value according to your application behaviour and memory needs. - * - * Note: a size threshold of 512 guarantees that newly created dictionaries - * will be allocated from preallocated memory pools on 64-bit. - * - * The following invariants must hold: - * 1) ALIGNMENT <= SMALL_REQUEST_THRESHOLD <= 512 - * 2) SMALL_REQUEST_THRESHOLD is evenly divisible by ALIGNMENT - * - * Although not required, for better performance and space efficiency, - * it is recommended that SMALL_REQUEST_THRESHOLD is set to a power of 2. - */ -#define SMALL_REQUEST_THRESHOLD 512 -#define NB_SMALL_SIZE_CLASSES (SMALL_REQUEST_THRESHOLD / ALIGNMENT) - -#if NB_SMALL_SIZE_CLASSES > 64 -#error "NB_SMALL_SIZE_CLASSES should be less than 64" -#endif /* NB_SMALL_SIZE_CLASSES > 64 */ - -/* - * The system's VMM page size can be obtained on most unices with a - * getpagesize() call or deduced from various header files. To make - * things simpler, we assume that it is 4K, which is OK for most systems. - * It is probably better if this is the native page size, but it doesn't - * have to be. In theory, if SYSTEM_PAGE_SIZE is larger than the native page - * size, then `POOL_ADDR(p)->arenaindex' could rarely cause a segmentation - * violation fault. 4K is apparently OK for all the platforms that python - * currently targets. - */ -#define SYSTEM_PAGE_SIZE (4 * 1024) -#define SYSTEM_PAGE_SIZE_MASK (SYSTEM_PAGE_SIZE - 1) - -/* - * Maximum amount of memory managed by the allocator for small requests. - */ -#ifdef WITH_MEMORY_LIMITS -#ifndef SMALL_MEMORY_LIMIT -#define SMALL_MEMORY_LIMIT (64 * 1024 * 1024) /* 64 MiB -- more? */ -#endif -#endif - -/* - * The allocator sub-allocates blocks of memory (called arenas) aligned - * on a page boundary. This is a reserved virtual address space for the - * current process (obtained through a malloc()/mmap() call). In no way this - * means that the memory arenas will be used entirely. A malloc() is - * usually an address range reservation for bytes, unless all pages within - * this space are referenced subsequently. So malloc'ing big blocks and not - * using them does not mean "wasting memory". It's an addressable range - * wastage... - * - * Arenas are allocated with mmap() on systems supporting anonymous memory - * mappings to reduce heap fragmentation. - */ -#define ARENA_SIZE (256 << 10) /* 256 KiB */ - -#ifdef WITH_MEMORY_LIMITS -#define MAX_ARENAS (SMALL_MEMORY_LIMIT / ARENA_SIZE) -#endif - -/* - * Size of the pools used for small blocks. Should be a power of 2, - * between 1K and SYSTEM_PAGE_SIZE, that is: 1k, 2k, 4k. - */ -#define POOL_SIZE SYSTEM_PAGE_SIZE /* must be 2^N */ -#define POOL_SIZE_MASK SYSTEM_PAGE_SIZE_MASK - -/* - * -- End of tunable settings section -- - */ - -/*==========================================================================*/ - -/* - * Locking - * - * To reduce lock contention, it would probably be better to refine the - * crude function locking with per size class locking. I'm not positive - * however, whether it's worth switching to such locking policy because - * of the performance penalty it might introduce. - * - * The following macros describe the simplest (should also be the fastest) - * lock object on a particular platform and the init/fini/lock/unlock - * operations on it. The locks defined here are not expected to be recursive - * because it is assumed that they will always be called in the order: - * INIT, [LOCK, UNLOCK]*, FINI. - */ - -/* - * Python's threads are serialized, so object malloc locking is disabled. - */ -#define SIMPLELOCK_DECL(lock) /* simple lock declaration */ -#define SIMPLELOCK_INIT(lock) /* allocate (if needed) and initialize */ -#define SIMPLELOCK_FINI(lock) /* free/destroy an existing lock */ -#define SIMPLELOCK_LOCK(lock) /* acquire released lock */ -#define SIMPLELOCK_UNLOCK(lock) /* release acquired lock */ - -/* When you say memory, my mind reasons in terms of (pointers to) blocks */ -typedef uint8_t pyblock; - -/* Pool for small blocks. */ -struct pool_header { - union { pyblock *_padding; - unsigned int count; } ref; /* number of allocated blocks */ - pyblock *freeblock; /* pool's free list head */ - struct pool_header *nextpool; /* next pool of this size class */ - struct pool_header *prevpool; /* previous pool "" */ - unsigned int arenaindex; /* index into arenas of base adr */ - unsigned int szidx; /* block size class index */ - unsigned int nextoffset; /* bytes to virgin block */ - unsigned int maxnextoffset; /* largest valid nextoffset */ -}; - -typedef struct pool_header *poolp; - -/* Record keeping for arenas. */ -struct arena_object { - /* The address of the arena, as returned by malloc. Note that 0 - * will never be returned by a successful malloc, and is used - * here to mark an arena_object that doesn't correspond to an - * allocated arena. - */ - uintptr_t address; - - /* Pool-aligned pointer to the next pool to be carved off. */ - pyblock* pool_address; - - /* The number of available pools in the arena: free pools + never- - * allocated pools. - */ - unsigned int nfreepools; - - /* The total number of pools in the arena, whether or not available. */ - unsigned int ntotalpools; - - /* Singly-linked list of available pools. */ - struct pool_header* freepools; - - /* Whenever this arena_object is not associated with an allocated - * arena, the nextarena member is used to link all unassociated - * arena_objects in the singly-linked `unused_arena_objects` list. - * The prevarena member is unused in this case. - * - * When this arena_object is associated with an allocated arena - * with at least one available pool, both members are used in the - * doubly-linked `usable_arenas` list, which is maintained in - * increasing order of `nfreepools` values. - * - * Else this arena_object is associated with an allocated arena - * all of whose pools are in use. `nextarena` and `prevarena` - * are both meaningless in this case. - */ - struct arena_object* nextarena; - struct arena_object* prevarena; -}; - -#define POOL_OVERHEAD _Py_SIZE_ROUND_UP(sizeof(struct pool_header), ALIGNMENT) - -#define DUMMY_SIZE_IDX 0xffff /* size class of newly cached pools */ - -/* Round pointer P down to the closest pool-aligned address <= P, as a poolp */ -#define POOL_ADDR(P) ((poolp)_Py_ALIGN_DOWN((P), POOL_SIZE)) - -/* Return total number of blocks in pool of size index I, as a uint. */ -#define NUMBLOCKS(I) \ - ((unsigned int)(POOL_SIZE - POOL_OVERHEAD) / INDEX2SIZE(I)) - -/*==========================================================================*/ - -/* - * This malloc lock - */ -SIMPLELOCK_DECL(_malloc_lock) -#define LOCK() SIMPLELOCK_LOCK(_malloc_lock) -#define UNLOCK() SIMPLELOCK_UNLOCK(_malloc_lock) -#define LOCK_INIT() SIMPLELOCK_INIT(_malloc_lock) -#define LOCK_FINI() SIMPLELOCK_FINI(_malloc_lock) - -/* - * Pool table -- headed, circular, doubly-linked lists of partially used pools. - -This is involved. For an index i, usedpools[i+i] is the header for a list of -all partially used pools holding small blocks with "size class idx" i. So -usedpools[0] corresponds to blocks of size 8, usedpools[2] to blocks of size -16, and so on: index 2*i <-> blocks of size (i+1)<freeblock points to -the start of a singly-linked list of free blocks within the pool. When a -block is freed, it's inserted at the front of its pool's freeblock list. Note -that the available blocks in a pool are *not* linked all together when a pool -is initialized. Instead only "the first two" (lowest addresses) blocks are -set up, returning the first such block, and setting pool->freeblock to a -one-block list holding the second such block. This is consistent with that -pymalloc strives at all levels (arena, pool, and block) never to touch a piece -of memory until it's actually needed. - -So long as a pool is in the used state, we're certain there *is* a block -available for allocating, and pool->freeblock is not NULL. If pool->freeblock -points to the end of the free list before we've carved the entire pool into -blocks, that means we simply haven't yet gotten to one of the higher-address -blocks. The offset from the pool_header to the start of "the next" virgin -block is stored in the pool_header nextoffset member, and the largest value -of nextoffset that makes sense is stored in the maxnextoffset member when a -pool is initialized. All the blocks in a pool have been passed out at least -once when and only when nextoffset > maxnextoffset. - - -Major obscurity: While the usedpools vector is declared to have poolp -entries, it doesn't really. It really contains two pointers per (conceptual) -poolp entry, the nextpool and prevpool members of a pool_header. The -excruciating initialization code below fools C so that - - usedpool[i+i] - -"acts like" a genuine poolp, but only so long as you only reference its -nextpool and prevpool members. The "- 2*sizeof(block *)" gibberish is -compensating for that a pool_header's nextpool and prevpool members -immediately follow a pool_header's first two members: - - union { block *_padding; - uint count; } ref; - block *freeblock; - -each of which consume sizeof(block *) bytes. So what usedpools[i+i] really -contains is a fudged-up pointer p such that *if* C believes it's a poolp -pointer, then p->nextpool and p->prevpool are both p (meaning that the headed -circular list is empty). - -It's unclear why the usedpools setup is so convoluted. It could be to -minimize the amount of cache required to hold this heavily-referenced table -(which only *needs* the two interpool pointer members of a pool_header). OTOH, -referencing code has to remember to "double the index" and doing so isn't -free, usedpools[0] isn't a strictly legal pointer, and we're crucially relying -on that C doesn't insert any padding anywhere in a pool_header at or before -the prevpool member. -**************************************************************************** */ - -#define MAX_POOLS (2 * ((NB_SMALL_SIZE_CLASSES + 7) / 8) * 8) - -/*========================================================================== -Arena management. - -`arenas` is a vector of arena_objects. It contains maxarenas entries, some of -which may not be currently used (== they're arena_objects that aren't -currently associated with an allocated arena). Note that arenas proper are -separately malloc'ed. - -Prior to Python 2.5, arenas were never free()'ed. Starting with Python 2.5, -we do try to free() arenas, and use some mild heuristic strategies to increase -the likelihood that arenas eventually can be freed. - -unused_arena_objects - - This is a singly-linked list of the arena_objects that are currently not - being used (no arena is associated with them). Objects are taken off the - head of the list in new_arena(), and are pushed on the head of the list in - PyObject_Free() when the arena is empty. Key invariant: an arena_object - is on this list if and only if its .address member is 0. - -usable_arenas - - This is a doubly-linked list of the arena_objects associated with arenas - that have pools available. These pools are either waiting to be reused, - or have not been used before. The list is sorted to have the most- - allocated arenas first (ascending order based on the nfreepools member). - This means that the next allocation will come from a heavily used arena, - which gives the nearly empty arenas a chance to be returned to the system. - In my unscientific tests this dramatically improved the number of arenas - that could be freed. - -Note that an arena_object associated with an arena all of whose pools are -currently in use isn't on either list. -*/ - -/* How many arena_objects do we initially allocate? - * 16 = can allocate 16 arenas = 16 * ARENA_SIZE = 4 MiB before growing the - * `arenas` vector. - */ -#define INITIAL_ARENA_OBJECTS 16 - -#endif /* Py_INTERNAL_PYMALLOC_H */ diff --git a/Include/internal/pystate.h b/Include/internal/pystate.h index 67b4a516a767c9..7056e105ff7dcb 100644 --- a/Include/internal/pystate.h +++ b/Include/internal/pystate.h @@ -64,9 +64,7 @@ typedef struct pyruntimestate { int nexitfuncs; void (*pyexitfunc)(void); - struct _pyobj_runtime_state obj; struct _gc_runtime_state gc; - struct _pymem_runtime_state mem; struct _warnings_runtime_state warnings; struct _ceval_runtime_state ceval; struct _gilstate_runtime_state gilstate; diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py index bb5b2a3b9f0d73..2fe0feca5a3ca4 100644 --- a/Lib/test/test_capi.py +++ b/Lib/test/test_capi.py @@ -593,6 +593,16 @@ def test_forced_io_encoding(self): self.maxDiff = None self.assertEqual(out.strip(), expected_output) + def test_pre_initialization_api(self): + """ + Checks the few parts of the C-API that work before the runtine + is initialized (via Py_Initialize()). + """ + env = dict(os.environ, PYTHONPATH=os.pathsep.join(sys.path)) + out, err = self.run_embedded_interpreter("pre_initialization_api", env=env) + self.assertEqual(out, '') + self.assertEqual(err, '') + class SkipitemTest(unittest.TestCase): diff --git a/Makefile.pre.in b/Makefile.pre.in index 566afba2538ac0..d196d5f838e888 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1015,7 +1015,6 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/ceval.h \ $(srcdir)/Include/internal/gil.h \ $(srcdir)/Include/internal/mem.h \ - $(srcdir)/Include/internal/pymalloc.h \ $(srcdir)/Include/internal/pystate.h \ $(srcdir)/Include/internal/warnings.h \ $(DTRACE_HEADERS) diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-11-24-01-13-58.bpo-32096.CQTHXJ.rst b/Misc/NEWS.d/next/Core and Builtins/2017-11-24-01-13-58.bpo-32096.CQTHXJ.rst new file mode 100644 index 00000000000000..d2a770b9375fa0 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2017-11-24-01-13-58.bpo-32096.CQTHXJ.rst @@ -0,0 +1,4 @@ +Revert memory allocator changes in the C API: move structures back from +_PyRuntime to Objects/obmalloc.c. The memory allocators are once again initialized +statically, and so PyMem_RawMalloc() and Py_DecodeLocale() can be +called before _PyRuntime_Initialize(). diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index 7c6973ec035f10..96a451ee498a16 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -1,6 +1,4 @@ #include "Python.h" -#include "internal/mem.h" -#include "internal/pystate.h" #include @@ -180,24 +178,39 @@ static struct { #define PYDBG_FUNCS \ _PyMem_DebugMalloc, _PyMem_DebugCalloc, _PyMem_DebugRealloc, _PyMem_DebugFree +static PyMemAllocatorEx _PyMem_Raw = { +#ifdef Py_DEBUG + &_PyMem_Debug.raw, PYRAWDBG_FUNCS +#else + NULL, PYRAW_FUNCS +#endif + }; -#define _PyMem_Raw _PyRuntime.mem.allocators.raw - -#define _PyMem _PyRuntime.mem.allocators.mem +static PyMemAllocatorEx _PyMem = { +#ifdef Py_DEBUG + &_PyMem_Debug.mem, PYDBG_FUNCS +#else + NULL, PYMEM_FUNCS +#endif + }; -#define _PyObject _PyRuntime.mem.allocators.obj +static PyMemAllocatorEx _PyObject = { +#ifdef Py_DEBUG + &_PyMem_Debug.obj, PYDBG_FUNCS +#else + NULL, PYOBJ_FUNCS +#endif + }; void _PyMem_GetDefaultRawAllocator(PyMemAllocatorEx *alloc_p) { - PyMemAllocatorEx pymem_raw = { #ifdef Py_DEBUG - &_PyMem_Debug.raw, PYRAWDBG_FUNCS + PyMemAllocatorEx alloc = {&_PyMem_Debug.raw, PYDBG_FUNCS}; #else - NULL, PYRAW_FUNCS + PyMemAllocatorEx alloc = {NULL, PYRAW_FUNCS}; #endif - }; - *alloc_p = pymem_raw; + *alloc_p = alloc; } int @@ -259,62 +272,22 @@ _PyMem_SetupAllocators(const char *opt) return 0; } +#undef PYRAW_FUNCS +#undef PYMEM_FUNCS +#undef PYOBJ_FUNCS +#undef PYRAWDBG_FUNCS +#undef PYDBG_FUNCS -void -_PyObject_Initialize(struct _pyobj_runtime_state *state) -{ - PyObjectArenaAllocator _PyObject_Arena = {NULL, +static PyObjectArenaAllocator _PyObject_Arena = {NULL, #ifdef MS_WINDOWS - _PyObject_ArenaVirtualAlloc, _PyObject_ArenaVirtualFree + _PyObject_ArenaVirtualAlloc, _PyObject_ArenaVirtualFree #elif defined(ARENAS_USE_MMAP) - _PyObject_ArenaMmap, _PyObject_ArenaMunmap + _PyObject_ArenaMmap, _PyObject_ArenaMunmap #else - _PyObject_ArenaMalloc, _PyObject_ArenaFree + _PyObject_ArenaMalloc, _PyObject_ArenaFree #endif }; - state->allocator_arenas = _PyObject_Arena; -} - - -void -_PyMem_Initialize(struct _pymem_runtime_state *state) -{ - PyMemAllocatorEx pymem = { -#ifdef Py_DEBUG - &_PyMem_Debug.mem, PYDBG_FUNCS -#else - NULL, PYMEM_FUNCS -#endif - }; - PyMemAllocatorEx pyobject = { -#ifdef Py_DEBUG - &_PyMem_Debug.obj, PYDBG_FUNCS -#else - NULL, PYOBJ_FUNCS -#endif - }; - - _PyMem_GetDefaultRawAllocator(&state->allocators.raw); - state->allocators.mem = pymem; - state->allocators.obj = pyobject; - -#ifdef WITH_PYMALLOC - Py_BUILD_ASSERT(NB_SMALL_SIZE_CLASSES == 64); - - for (int i = 0; i < 8; i++) { - for (int j = 0; j < 8; j++) { - int x = i * 8 + j; - poolp *addr = &(state->usedpools[2*(x)]); - poolp val = (poolp)((uint8_t *)addr - 2*sizeof(pyblock *)); - state->usedpools[x * 2] = val; - state->usedpools[x * 2 + 1] = val; - }; - }; -#endif /* WITH_PYMALLOC */ -} - - #ifdef WITH_PYMALLOC static int _PyMem_DebugEnabled(void) @@ -401,13 +374,13 @@ PyMem_SetAllocator(PyMemAllocatorDomain domain, PyMemAllocatorEx *allocator) void PyObject_GetArenaAllocator(PyObjectArenaAllocator *allocator) { - *allocator = _PyRuntime.obj.allocator_arenas; + *allocator = _PyObject_Arena; } void PyObject_SetArenaAllocator(PyObjectArenaAllocator *allocator) { - _PyRuntime.obj.allocator_arenas = *allocator; + _PyObject_Arena = *allocator; } void * @@ -442,8 +415,7 @@ PyMem_RawRealloc(void *ptr, size_t new_size) return _PyMem_Raw.realloc(_PyMem_Raw.ctx, ptr, new_size); } -void -PyMem_RawFree(void *ptr) +void PyMem_RawFree(void *ptr) { _PyMem_Raw.free(_PyMem_Raw.ctx, ptr); } @@ -563,10 +535,497 @@ static int running_on_valgrind = -1; #endif +/* An object allocator for Python. + + Here is an introduction to the layers of the Python memory architecture, + showing where the object allocator is actually used (layer +2), It is + called for every object allocation and deallocation (PyObject_New/Del), + unless the object-specific allocators implement a proprietary allocation + scheme (ex.: ints use a simple free list). This is also the place where + the cyclic garbage collector operates selectively on container objects. + + + Object-specific allocators + _____ ______ ______ ________ + [ int ] [ dict ] [ list ] ... [ string ] Python core | ++3 | <----- Object-specific memory -----> | <-- Non-object memory --> | + _______________________________ | | + [ Python's object allocator ] | | ++2 | ####### Object memory ####### | <------ Internal buffers ------> | + ______________________________________________________________ | + [ Python's raw memory allocator (PyMem_ API) ] | ++1 | <----- Python memory (under PyMem manager's control) ------> | | + __________________________________________________________________ + [ Underlying general-purpose allocator (ex: C library malloc) ] + 0 | <------ Virtual memory allocated for the python process -------> | + + ========================================================================= + _______________________________________________________________________ + [ OS-specific Virtual Memory Manager (VMM) ] +-1 | <--- Kernel dynamic storage allocation & management (page-based) ---> | + __________________________________ __________________________________ + [ ] [ ] +-2 | <-- Physical memory: ROM/RAM --> | | <-- Secondary storage (swap) --> | + +*/ +/*==========================================================================*/ + +/* A fast, special-purpose memory allocator for small blocks, to be used + on top of a general-purpose malloc -- heavily based on previous art. */ + +/* Vladimir Marangozov -- August 2000 */ + +/* + * "Memory management is where the rubber meets the road -- if we do the wrong + * thing at any level, the results will not be good. And if we don't make the + * levels work well together, we are in serious trouble." (1) + * + * (1) Paul R. Wilson, Mark S. Johnstone, Michael Neely, and David Boles, + * "Dynamic Storage Allocation: A Survey and Critical Review", + * in Proc. 1995 Int'l. Workshop on Memory Management, September 1995. + */ + +/* #undef WITH_MEMORY_LIMITS */ /* disable mem limit checks */ + +/*==========================================================================*/ + +/* + * Allocation strategy abstract: + * + * For small requests, the allocator sub-allocates blocks of memory. + * Requests greater than SMALL_REQUEST_THRESHOLD bytes are routed to the + * system's allocator. + * + * Small requests are grouped in size classes spaced 8 bytes apart, due + * to the required valid alignment of the returned address. Requests of + * a particular size are serviced from memory pools of 4K (one VMM page). + * Pools are fragmented on demand and contain free lists of blocks of one + * particular size class. In other words, there is a fixed-size allocator + * for each size class. Free pools are shared by the different allocators + * thus minimizing the space reserved for a particular size class. + * + * This allocation strategy is a variant of what is known as "simple + * segregated storage based on array of free lists". The main drawback of + * simple segregated storage is that we might end up with lot of reserved + * memory for the different free lists, which degenerate in time. To avoid + * this, we partition each free list in pools and we share dynamically the + * reserved space between all free lists. This technique is quite efficient + * for memory intensive programs which allocate mainly small-sized blocks. + * + * For small requests we have the following table: + * + * Request in bytes Size of allocated block Size class idx + * ---------------------------------------------------------------- + * 1-8 8 0 + * 9-16 16 1 + * 17-24 24 2 + * 25-32 32 3 + * 33-40 40 4 + * 41-48 48 5 + * 49-56 56 6 + * 57-64 64 7 + * 65-72 72 8 + * ... ... ... + * 497-504 504 62 + * 505-512 512 63 + * + * 0, SMALL_REQUEST_THRESHOLD + 1 and up: routed to the underlying + * allocator. + */ + +/*==========================================================================*/ + +/* + * -- Main tunable settings section -- + */ + +/* + * Alignment of addresses returned to the user. 8-bytes alignment works + * on most current architectures (with 32-bit or 64-bit address busses). + * The alignment value is also used for grouping small requests in size + * classes spaced ALIGNMENT bytes apart. + * + * You shouldn't change this unless you know what you are doing. + */ +#define ALIGNMENT 8 /* must be 2^N */ +#define ALIGNMENT_SHIFT 3 + +/* Return the number of bytes in size class I, as a uint. */ +#define INDEX2SIZE(I) (((uint)(I) + 1) << ALIGNMENT_SHIFT) + +/* + * Max size threshold below which malloc requests are considered to be + * small enough in order to use preallocated memory pools. You can tune + * this value according to your application behaviour and memory needs. + * + * Note: a size threshold of 512 guarantees that newly created dictionaries + * will be allocated from preallocated memory pools on 64-bit. + * + * The following invariants must hold: + * 1) ALIGNMENT <= SMALL_REQUEST_THRESHOLD <= 512 + * 2) SMALL_REQUEST_THRESHOLD is evenly divisible by ALIGNMENT + * + * Although not required, for better performance and space efficiency, + * it is recommended that SMALL_REQUEST_THRESHOLD is set to a power of 2. + */ +#define SMALL_REQUEST_THRESHOLD 512 +#define NB_SMALL_SIZE_CLASSES (SMALL_REQUEST_THRESHOLD / ALIGNMENT) + +/* + * The system's VMM page size can be obtained on most unices with a + * getpagesize() call or deduced from various header files. To make + * things simpler, we assume that it is 4K, which is OK for most systems. + * It is probably better if this is the native page size, but it doesn't + * have to be. In theory, if SYSTEM_PAGE_SIZE is larger than the native page + * size, then `POOL_ADDR(p)->arenaindex' could rarely cause a segmentation + * violation fault. 4K is apparently OK for all the platforms that python + * currently targets. + */ +#define SYSTEM_PAGE_SIZE (4 * 1024) +#define SYSTEM_PAGE_SIZE_MASK (SYSTEM_PAGE_SIZE - 1) + +/* + * Maximum amount of memory managed by the allocator for small requests. + */ +#ifdef WITH_MEMORY_LIMITS +#ifndef SMALL_MEMORY_LIMIT +#define SMALL_MEMORY_LIMIT (64 * 1024 * 1024) /* 64 MB -- more? */ +#endif +#endif + +/* + * The allocator sub-allocates blocks of memory (called arenas) aligned + * on a page boundary. This is a reserved virtual address space for the + * current process (obtained through a malloc()/mmap() call). In no way this + * means that the memory arenas will be used entirely. A malloc() is + * usually an address range reservation for bytes, unless all pages within + * this space are referenced subsequently. So malloc'ing big blocks and not + * using them does not mean "wasting memory". It's an addressable range + * wastage... + * + * Arenas are allocated with mmap() on systems supporting anonymous memory + * mappings to reduce heap fragmentation. + */ +#define ARENA_SIZE (256 << 10) /* 256KB */ + +#ifdef WITH_MEMORY_LIMITS +#define MAX_ARENAS (SMALL_MEMORY_LIMIT / ARENA_SIZE) +#endif + +/* + * Size of the pools used for small blocks. Should be a power of 2, + * between 1K and SYSTEM_PAGE_SIZE, that is: 1k, 2k, 4k. + */ +#define POOL_SIZE SYSTEM_PAGE_SIZE /* must be 2^N */ +#define POOL_SIZE_MASK SYSTEM_PAGE_SIZE_MASK + +/* + * -- End of tunable settings section -- + */ + +/*==========================================================================*/ + +/* + * Locking + * + * To reduce lock contention, it would probably be better to refine the + * crude function locking with per size class locking. I'm not positive + * however, whether it's worth switching to such locking policy because + * of the performance penalty it might introduce. + * + * The following macros describe the simplest (should also be the fastest) + * lock object on a particular platform and the init/fini/lock/unlock + * operations on it. The locks defined here are not expected to be recursive + * because it is assumed that they will always be called in the order: + * INIT, [LOCK, UNLOCK]*, FINI. + */ + +/* + * Python's threads are serialized, so object malloc locking is disabled. + */ +#define SIMPLELOCK_DECL(lock) /* simple lock declaration */ +#define SIMPLELOCK_INIT(lock) /* allocate (if needed) and initialize */ +#define SIMPLELOCK_FINI(lock) /* free/destroy an existing lock */ +#define SIMPLELOCK_LOCK(lock) /* acquire released lock */ +#define SIMPLELOCK_UNLOCK(lock) /* release acquired lock */ + +/* When you say memory, my mind reasons in terms of (pointers to) blocks */ +typedef uint8_t block; + +/* Pool for small blocks. */ +struct pool_header { + union { block *_padding; + uint count; } ref; /* number of allocated blocks */ + block *freeblock; /* pool's free list head */ + struct pool_header *nextpool; /* next pool of this size class */ + struct pool_header *prevpool; /* previous pool "" */ + uint arenaindex; /* index into arenas of base adr */ + uint szidx; /* block size class index */ + uint nextoffset; /* bytes to virgin block */ + uint maxnextoffset; /* largest valid nextoffset */ +}; + +typedef struct pool_header *poolp; + +/* Record keeping for arenas. */ +struct arena_object { + /* The address of the arena, as returned by malloc. Note that 0 + * will never be returned by a successful malloc, and is used + * here to mark an arena_object that doesn't correspond to an + * allocated arena. + */ + uintptr_t address; + + /* Pool-aligned pointer to the next pool to be carved off. */ + block* pool_address; + + /* The number of available pools in the arena: free pools + never- + * allocated pools. + */ + uint nfreepools; + + /* The total number of pools in the arena, whether or not available. */ + uint ntotalpools; + + /* Singly-linked list of available pools. */ + struct pool_header* freepools; + + /* Whenever this arena_object is not associated with an allocated + * arena, the nextarena member is used to link all unassociated + * arena_objects in the singly-linked `unused_arena_objects` list. + * The prevarena member is unused in this case. + * + * When this arena_object is associated with an allocated arena + * with at least one available pool, both members are used in the + * doubly-linked `usable_arenas` list, which is maintained in + * increasing order of `nfreepools` values. + * + * Else this arena_object is associated with an allocated arena + * all of whose pools are in use. `nextarena` and `prevarena` + * are both meaningless in this case. + */ + struct arena_object* nextarena; + struct arena_object* prevarena; +}; + +#define POOL_OVERHEAD _Py_SIZE_ROUND_UP(sizeof(struct pool_header), ALIGNMENT) + +#define DUMMY_SIZE_IDX 0xffff /* size class of newly cached pools */ + +/* Round pointer P down to the closest pool-aligned address <= P, as a poolp */ +#define POOL_ADDR(P) ((poolp)_Py_ALIGN_DOWN((P), POOL_SIZE)) + +/* Return total number of blocks in pool of size index I, as a uint. */ +#define NUMBLOCKS(I) ((uint)(POOL_SIZE - POOL_OVERHEAD) / INDEX2SIZE(I)) + +/*==========================================================================*/ + +/* + * This malloc lock + */ +SIMPLELOCK_DECL(_malloc_lock) +#define LOCK() SIMPLELOCK_LOCK(_malloc_lock) +#define UNLOCK() SIMPLELOCK_UNLOCK(_malloc_lock) +#define LOCK_INIT() SIMPLELOCK_INIT(_malloc_lock) +#define LOCK_FINI() SIMPLELOCK_FINI(_malloc_lock) + +/* + * Pool table -- headed, circular, doubly-linked lists of partially used pools. + +This is involved. For an index i, usedpools[i+i] is the header for a list of +all partially used pools holding small blocks with "size class idx" i. So +usedpools[0] corresponds to blocks of size 8, usedpools[2] to blocks of size +16, and so on: index 2*i <-> blocks of size (i+1)<freeblock points to +the start of a singly-linked list of free blocks within the pool. When a +block is freed, it's inserted at the front of its pool's freeblock list. Note +that the available blocks in a pool are *not* linked all together when a pool +is initialized. Instead only "the first two" (lowest addresses) blocks are +set up, returning the first such block, and setting pool->freeblock to a +one-block list holding the second such block. This is consistent with that +pymalloc strives at all levels (arena, pool, and block) never to touch a piece +of memory until it's actually needed. + +So long as a pool is in the used state, we're certain there *is* a block +available for allocating, and pool->freeblock is not NULL. If pool->freeblock +points to the end of the free list before we've carved the entire pool into +blocks, that means we simply haven't yet gotten to one of the higher-address +blocks. The offset from the pool_header to the start of "the next" virgin +block is stored in the pool_header nextoffset member, and the largest value +of nextoffset that makes sense is stored in the maxnextoffset member when a +pool is initialized. All the blocks in a pool have been passed out at least +once when and only when nextoffset > maxnextoffset. + + +Major obscurity: While the usedpools vector is declared to have poolp +entries, it doesn't really. It really contains two pointers per (conceptual) +poolp entry, the nextpool and prevpool members of a pool_header. The +excruciating initialization code below fools C so that + + usedpool[i+i] + +"acts like" a genuine poolp, but only so long as you only reference its +nextpool and prevpool members. The "- 2*sizeof(block *)" gibberish is +compensating for that a pool_header's nextpool and prevpool members +immediately follow a pool_header's first two members: + + union { block *_padding; + uint count; } ref; + block *freeblock; + +each of which consume sizeof(block *) bytes. So what usedpools[i+i] really +contains is a fudged-up pointer p such that *if* C believes it's a poolp +pointer, then p->nextpool and p->prevpool are both p (meaning that the headed +circular list is empty). + +It's unclear why the usedpools setup is so convoluted. It could be to +minimize the amount of cache required to hold this heavily-referenced table +(which only *needs* the two interpool pointer members of a pool_header). OTOH, +referencing code has to remember to "double the index" and doing so isn't +free, usedpools[0] isn't a strictly legal pointer, and we're crucially relying +on that C doesn't insert any padding anywhere in a pool_header at or before +the prevpool member. +**************************************************************************** */ + +#define PTA(x) ((poolp )((uint8_t *)&(usedpools[2*(x)]) - 2*sizeof(block *))) +#define PT(x) PTA(x), PTA(x) + +static poolp usedpools[2 * ((NB_SMALL_SIZE_CLASSES + 7) / 8) * 8] = { + PT(0), PT(1), PT(2), PT(3), PT(4), PT(5), PT(6), PT(7) +#if NB_SMALL_SIZE_CLASSES > 8 + , PT(8), PT(9), PT(10), PT(11), PT(12), PT(13), PT(14), PT(15) +#if NB_SMALL_SIZE_CLASSES > 16 + , PT(16), PT(17), PT(18), PT(19), PT(20), PT(21), PT(22), PT(23) +#if NB_SMALL_SIZE_CLASSES > 24 + , PT(24), PT(25), PT(26), PT(27), PT(28), PT(29), PT(30), PT(31) +#if NB_SMALL_SIZE_CLASSES > 32 + , PT(32), PT(33), PT(34), PT(35), PT(36), PT(37), PT(38), PT(39) +#if NB_SMALL_SIZE_CLASSES > 40 + , PT(40), PT(41), PT(42), PT(43), PT(44), PT(45), PT(46), PT(47) +#if NB_SMALL_SIZE_CLASSES > 48 + , PT(48), PT(49), PT(50), PT(51), PT(52), PT(53), PT(54), PT(55) +#if NB_SMALL_SIZE_CLASSES > 56 + , PT(56), PT(57), PT(58), PT(59), PT(60), PT(61), PT(62), PT(63) +#if NB_SMALL_SIZE_CLASSES > 64 +#error "NB_SMALL_SIZE_CLASSES should be less than 64" +#endif /* NB_SMALL_SIZE_CLASSES > 64 */ +#endif /* NB_SMALL_SIZE_CLASSES > 56 */ +#endif /* NB_SMALL_SIZE_CLASSES > 48 */ +#endif /* NB_SMALL_SIZE_CLASSES > 40 */ +#endif /* NB_SMALL_SIZE_CLASSES > 32 */ +#endif /* NB_SMALL_SIZE_CLASSES > 24 */ +#endif /* NB_SMALL_SIZE_CLASSES > 16 */ +#endif /* NB_SMALL_SIZE_CLASSES > 8 */ +}; + +/*========================================================================== +Arena management. + +`arenas` is a vector of arena_objects. It contains maxarenas entries, some of +which may not be currently used (== they're arena_objects that aren't +currently associated with an allocated arena). Note that arenas proper are +separately malloc'ed. + +Prior to Python 2.5, arenas were never free()'ed. Starting with Python 2.5, +we do try to free() arenas, and use some mild heuristic strategies to increase +the likelihood that arenas eventually can be freed. + +unused_arena_objects + + This is a singly-linked list of the arena_objects that are currently not + being used (no arena is associated with them). Objects are taken off the + head of the list in new_arena(), and are pushed on the head of the list in + PyObject_Free() when the arena is empty. Key invariant: an arena_object + is on this list if and only if its .address member is 0. + +usable_arenas + + This is a doubly-linked list of the arena_objects associated with arenas + that have pools available. These pools are either waiting to be reused, + or have not been used before. The list is sorted to have the most- + allocated arenas first (ascending order based on the nfreepools member). + This means that the next allocation will come from a heavily used arena, + which gives the nearly empty arenas a chance to be returned to the system. + In my unscientific tests this dramatically improved the number of arenas + that could be freed. + +Note that an arena_object associated with an arena all of whose pools are +currently in use isn't on either list. +*/ + +/* Array of objects used to track chunks of memory (arenas). */ +static struct arena_object* arenas = NULL; +/* Number of slots currently allocated in the `arenas` vector. */ +static uint maxarenas = 0; + +/* The head of the singly-linked, NULL-terminated list of available + * arena_objects. + */ +static struct arena_object* unused_arena_objects = NULL; + +/* The head of the doubly-linked, NULL-terminated at each end, list of + * arena_objects associated with arenas that have pools available. + */ +static struct arena_object* usable_arenas = NULL; + +/* How many arena_objects do we initially allocate? + * 16 = can allocate 16 arenas = 16 * ARENA_SIZE = 4MB before growing the + * `arenas` vector. + */ +#define INITIAL_ARENA_OBJECTS 16 + +/* Number of arenas allocated that haven't been free()'d. */ +static size_t narenas_currently_allocated = 0; + +/* Total number of times malloc() called to allocate an arena. */ +static size_t ntimes_arena_allocated = 0; +/* High water mark (max value ever seen) for narenas_currently_allocated. */ +static size_t narenas_highwater = 0; + +static Py_ssize_t _Py_AllocatedBlocks = 0; + Py_ssize_t _Py_GetAllocatedBlocks(void) { - return _PyRuntime.mem.num_allocated_blocks; + return _Py_AllocatedBlocks; } @@ -590,7 +1049,7 @@ new_arena(void) if (debug_stats) _PyObject_DebugMallocStats(stderr); - if (_PyRuntime.mem.unused_arena_objects == NULL) { + if (unused_arena_objects == NULL) { uint i; uint numarenas; size_t nbytes; @@ -598,18 +1057,18 @@ new_arena(void) /* Double the number of arena objects on each allocation. * Note that it's possible for `numarenas` to overflow. */ - numarenas = _PyRuntime.mem.maxarenas ? _PyRuntime.mem.maxarenas << 1 : INITIAL_ARENA_OBJECTS; - if (numarenas <= _PyRuntime.mem.maxarenas) + numarenas = maxarenas ? maxarenas << 1 : INITIAL_ARENA_OBJECTS; + if (numarenas <= maxarenas) return NULL; /* overflow */ #if SIZEOF_SIZE_T <= SIZEOF_INT - if (numarenas > SIZE_MAX / sizeof(*_PyRuntime.mem.arenas)) + if (numarenas > SIZE_MAX / sizeof(*arenas)) return NULL; /* overflow */ #endif - nbytes = numarenas * sizeof(*_PyRuntime.mem.arenas); - arenaobj = (struct arena_object *)PyMem_RawRealloc(_PyRuntime.mem.arenas, nbytes); + nbytes = numarenas * sizeof(*arenas); + arenaobj = (struct arena_object *)PyMem_RawRealloc(arenas, nbytes); if (arenaobj == NULL) return NULL; - _PyRuntime.mem.arenas = arenaobj; + arenas = arenaobj; /* We might need to fix pointers that were copied. However, * new_arena only gets called when all the pages in the @@ -617,45 +1076,45 @@ new_arena(void) * into the old array. Thus, we don't have to worry about * invalid pointers. Just to be sure, some asserts: */ - assert(_PyRuntime.mem.usable_arenas == NULL); - assert(_PyRuntime.mem.unused_arena_objects == NULL); + assert(usable_arenas == NULL); + assert(unused_arena_objects == NULL); /* Put the new arenas on the unused_arena_objects list. */ - for (i = _PyRuntime.mem.maxarenas; i < numarenas; ++i) { - _PyRuntime.mem.arenas[i].address = 0; /* mark as unassociated */ - _PyRuntime.mem.arenas[i].nextarena = i < numarenas - 1 ? - &_PyRuntime.mem.arenas[i+1] : NULL; + for (i = maxarenas; i < numarenas; ++i) { + arenas[i].address = 0; /* mark as unassociated */ + arenas[i].nextarena = i < numarenas - 1 ? + &arenas[i+1] : NULL; } /* Update globals. */ - _PyRuntime.mem.unused_arena_objects = &_PyRuntime.mem.arenas[_PyRuntime.mem.maxarenas]; - _PyRuntime.mem.maxarenas = numarenas; + unused_arena_objects = &arenas[maxarenas]; + maxarenas = numarenas; } /* Take the next available arena object off the head of the list. */ - assert(_PyRuntime.mem.unused_arena_objects != NULL); - arenaobj = _PyRuntime.mem.unused_arena_objects; - _PyRuntime.mem.unused_arena_objects = arenaobj->nextarena; + assert(unused_arena_objects != NULL); + arenaobj = unused_arena_objects; + unused_arena_objects = arenaobj->nextarena; assert(arenaobj->address == 0); - address = _PyRuntime.obj.allocator_arenas.alloc(_PyRuntime.obj.allocator_arenas.ctx, ARENA_SIZE); + address = _PyObject_Arena.alloc(_PyObject_Arena.ctx, ARENA_SIZE); if (address == NULL) { /* The allocation failed: return NULL after putting the * arenaobj back. */ - arenaobj->nextarena = _PyRuntime.mem.unused_arena_objects; - _PyRuntime.mem.unused_arena_objects = arenaobj; + arenaobj->nextarena = unused_arena_objects; + unused_arena_objects = arenaobj; return NULL; } arenaobj->address = (uintptr_t)address; - ++_PyRuntime.mem.narenas_currently_allocated; - ++_PyRuntime.mem.ntimes_arena_allocated; - if (_PyRuntime.mem.narenas_currently_allocated > _PyRuntime.mem.narenas_highwater) - _PyRuntime.mem.narenas_highwater = _PyRuntime.mem.narenas_currently_allocated; + ++narenas_currently_allocated; + ++ntimes_arena_allocated; + if (narenas_currently_allocated > narenas_highwater) + narenas_highwater = narenas_currently_allocated; arenaobj->freepools = NULL; /* pool_address <- first pool-aligned address in the arena nfreepools <- number of whole pools that fit after alignment */ - arenaobj->pool_address = (pyblock*)arenaobj->address; + arenaobj->pool_address = (block*)arenaobj->address; arenaobj->nfreepools = ARENA_SIZE / POOL_SIZE; assert(POOL_SIZE * arenaobj->nfreepools == ARENA_SIZE); excess = (uint)(arenaobj->address & POOL_SIZE_MASK); @@ -753,9 +1212,9 @@ address_in_range(void *p, poolp pool) // the GIL. The following dance forces the compiler to read pool->arenaindex // only once. uint arenaindex = *((volatile uint *)&pool->arenaindex); - return arenaindex < _PyRuntime.mem.maxarenas && - (uintptr_t)p - _PyRuntime.mem.arenas[arenaindex].address < ARENA_SIZE && - _PyRuntime.mem.arenas[arenaindex].address != 0; + return arenaindex < maxarenas && + (uintptr_t)p - arenas[arenaindex].address < ARENA_SIZE && + arenas[arenaindex].address != 0; } @@ -777,7 +1236,7 @@ address_in_range(void *p, poolp pool) static int pymalloc_alloc(void *ctx, void **ptr_p, size_t nbytes) { - pyblock *bp; + block *bp; poolp pool; poolp next; uint size; @@ -803,7 +1262,7 @@ pymalloc_alloc(void *ctx, void **ptr_p, size_t nbytes) * Most frequent paths first */ size = (uint)(nbytes - 1) >> ALIGNMENT_SHIFT; - pool = _PyRuntime.mem.usedpools[size + size]; + pool = usedpools[size + size]; if (pool != pool->nextpool) { /* * There is a used pool for this size class. @@ -812,7 +1271,7 @@ pymalloc_alloc(void *ctx, void **ptr_p, size_t nbytes) ++pool->ref.count; bp = pool->freeblock; assert(bp != NULL); - if ((pool->freeblock = *(pyblock **)bp) != NULL) { + if ((pool->freeblock = *(block **)bp) != NULL) { goto success; } @@ -821,10 +1280,10 @@ pymalloc_alloc(void *ctx, void **ptr_p, size_t nbytes) */ if (pool->nextoffset <= pool->maxnextoffset) { /* There is room for another block. */ - pool->freeblock = (pyblock*)pool + + pool->freeblock = (block*)pool + pool->nextoffset; pool->nextoffset += INDEX2SIZE(size); - *(pyblock **)(pool->freeblock) = NULL; + *(block **)(pool->freeblock) = NULL; goto success; } @@ -839,27 +1298,27 @@ pymalloc_alloc(void *ctx, void **ptr_p, size_t nbytes) /* There isn't a pool of the right size class immediately * available: use a free pool. */ - if (_PyRuntime.mem.usable_arenas == NULL) { + if (usable_arenas == NULL) { /* No arena has a free pool: allocate a new arena. */ #ifdef WITH_MEMORY_LIMITS - if (_PyRuntime.mem.narenas_currently_allocated >= MAX_ARENAS) { + if (narenas_currently_allocated >= MAX_ARENAS) { goto failed; } #endif - _PyRuntime.mem.usable_arenas = new_arena(); - if (_PyRuntime.mem.usable_arenas == NULL) { + usable_arenas = new_arena(); + if (usable_arenas == NULL) { goto failed; } - _PyRuntime.mem.usable_arenas->nextarena = - _PyRuntime.mem.usable_arenas->prevarena = NULL; + usable_arenas->nextarena = + usable_arenas->prevarena = NULL; } - assert(_PyRuntime.mem.usable_arenas->address != 0); + assert(usable_arenas->address != 0); /* Try to get a cached free pool. */ - pool = _PyRuntime.mem.usable_arenas->freepools; + pool = usable_arenas->freepools; if (pool != NULL) { /* Unlink from cached pools. */ - _PyRuntime.mem.usable_arenas->freepools = pool->nextpool; + usable_arenas->freepools = pool->nextpool; /* This arena already had the smallest nfreepools * value, so decreasing nfreepools doesn't change @@ -868,18 +1327,18 @@ pymalloc_alloc(void *ctx, void **ptr_p, size_t nbytes) * become wholly allocated, we need to remove its * arena_object from usable_arenas. */ - --_PyRuntime.mem.usable_arenas->nfreepools; - if (_PyRuntime.mem.usable_arenas->nfreepools == 0) { + --usable_arenas->nfreepools; + if (usable_arenas->nfreepools == 0) { /* Wholly allocated: remove. */ - assert(_PyRuntime.mem.usable_arenas->freepools == NULL); - assert(_PyRuntime.mem.usable_arenas->nextarena == NULL || - _PyRuntime.mem.usable_arenas->nextarena->prevarena == - _PyRuntime.mem.usable_arenas); - - _PyRuntime.mem.usable_arenas = _PyRuntime.mem.usable_arenas->nextarena; - if (_PyRuntime.mem.usable_arenas != NULL) { - _PyRuntime.mem.usable_arenas->prevarena = NULL; - assert(_PyRuntime.mem.usable_arenas->address != 0); + assert(usable_arenas->freepools == NULL); + assert(usable_arenas->nextarena == NULL || + usable_arenas->nextarena->prevarena == + usable_arenas); + + usable_arenas = usable_arenas->nextarena; + if (usable_arenas != NULL) { + usable_arenas->prevarena = NULL; + assert(usable_arenas->address != 0); } } else { @@ -888,15 +1347,15 @@ pymalloc_alloc(void *ctx, void **ptr_p, size_t nbytes) * off all the arena's pools for the first * time. */ - assert(_PyRuntime.mem.usable_arenas->freepools != NULL || - _PyRuntime.mem.usable_arenas->pool_address <= - (pyblock*)_PyRuntime.mem.usable_arenas->address + + assert(usable_arenas->freepools != NULL || + usable_arenas->pool_address <= + (block*)usable_arenas->address + ARENA_SIZE - POOL_SIZE); } init_pool: /* Frontlink to used pools. */ - next = _PyRuntime.mem.usedpools[size + size]; /* == prev */ + next = usedpools[size + size]; /* == prev */ pool->nextpool = next; pool->prevpool = next; next->nextpool = pool; @@ -909,7 +1368,7 @@ pymalloc_alloc(void *ctx, void **ptr_p, size_t nbytes) */ bp = pool->freeblock; assert(bp != NULL); - pool->freeblock = *(pyblock **)bp; + pool->freeblock = *(block **)bp; goto success; } /* @@ -919,35 +1378,35 @@ pymalloc_alloc(void *ctx, void **ptr_p, size_t nbytes) */ pool->szidx = size; size = INDEX2SIZE(size); - bp = (pyblock *)pool + POOL_OVERHEAD; + bp = (block *)pool + POOL_OVERHEAD; pool->nextoffset = POOL_OVERHEAD + (size << 1); pool->maxnextoffset = POOL_SIZE - size; pool->freeblock = bp + size; - *(pyblock **)(pool->freeblock) = NULL; + *(block **)(pool->freeblock) = NULL; goto success; } /* Carve off a new pool. */ - assert(_PyRuntime.mem.usable_arenas->nfreepools > 0); - assert(_PyRuntime.mem.usable_arenas->freepools == NULL); - pool = (poolp)_PyRuntime.mem.usable_arenas->pool_address; - assert((pyblock*)pool <= (pyblock*)_PyRuntime.mem.usable_arenas->address + + assert(usable_arenas->nfreepools > 0); + assert(usable_arenas->freepools == NULL); + pool = (poolp)usable_arenas->pool_address; + assert((block*)pool <= (block*)usable_arenas->address + ARENA_SIZE - POOL_SIZE); - pool->arenaindex = (uint)(_PyRuntime.mem.usable_arenas - _PyRuntime.mem.arenas); - assert(&_PyRuntime.mem.arenas[pool->arenaindex] == _PyRuntime.mem.usable_arenas); + pool->arenaindex = (uint)(usable_arenas - arenas); + assert(&arenas[pool->arenaindex] == usable_arenas); pool->szidx = DUMMY_SIZE_IDX; - _PyRuntime.mem.usable_arenas->pool_address += POOL_SIZE; - --_PyRuntime.mem.usable_arenas->nfreepools; + usable_arenas->pool_address += POOL_SIZE; + --usable_arenas->nfreepools; - if (_PyRuntime.mem.usable_arenas->nfreepools == 0) { - assert(_PyRuntime.mem.usable_arenas->nextarena == NULL || - _PyRuntime.mem.usable_arenas->nextarena->prevarena == - _PyRuntime.mem.usable_arenas); + if (usable_arenas->nfreepools == 0) { + assert(usable_arenas->nextarena == NULL || + usable_arenas->nextarena->prevarena == + usable_arenas); /* Unlink the arena: it is completely allocated. */ - _PyRuntime.mem.usable_arenas = _PyRuntime.mem.usable_arenas->nextarena; - if (_PyRuntime.mem.usable_arenas != NULL) { - _PyRuntime.mem.usable_arenas->prevarena = NULL; - assert(_PyRuntime.mem.usable_arenas->address != 0); + usable_arenas = usable_arenas->nextarena; + if (usable_arenas != NULL) { + usable_arenas->prevarena = NULL; + assert(usable_arenas->address != 0); } } @@ -970,13 +1429,13 @@ _PyObject_Malloc(void *ctx, size_t nbytes) { void* ptr; if (pymalloc_alloc(ctx, &ptr, nbytes)) { - _PyRuntime.mem.num_allocated_blocks++; + _Py_AllocatedBlocks++; return ptr; } ptr = PyMem_RawMalloc(nbytes); if (ptr != NULL) { - _PyRuntime.mem.num_allocated_blocks++; + _Py_AllocatedBlocks++; } return ptr; } @@ -992,13 +1451,13 @@ _PyObject_Calloc(void *ctx, size_t nelem, size_t elsize) if (pymalloc_alloc(ctx, &ptr, nbytes)) { memset(ptr, 0, nbytes); - _PyRuntime.mem.num_allocated_blocks++; + _Py_AllocatedBlocks++; return ptr; } ptr = PyMem_RawCalloc(nelem, elsize); if (ptr != NULL) { - _PyRuntime.mem.num_allocated_blocks++; + _Py_AllocatedBlocks++; } return ptr; } @@ -1011,7 +1470,7 @@ static int pymalloc_free(void *ctx, void *p) { poolp pool; - pyblock *lastfree; + block *lastfree; poolp next, prev; uint size; @@ -1038,8 +1497,8 @@ pymalloc_free(void *ctx, void *p) * list in any case). */ assert(pool->ref.count > 0); /* else it was empty */ - *(pyblock **)p = lastfree = pool->freeblock; - pool->freeblock = (pyblock *)p; + *(block **)p = lastfree = pool->freeblock; + pool->freeblock = (block *)p; if (!lastfree) { /* Pool was full, so doesn't currently live in any list: * link it to the front of the appropriate usedpools[] list. @@ -1050,7 +1509,7 @@ pymalloc_free(void *ctx, void *p) --pool->ref.count; assert(pool->ref.count > 0); /* else the pool is empty */ size = pool->szidx; - next = _PyRuntime.mem.usedpools[size + size]; + next = usedpools[size + size]; prev = next->prevpool; /* insert pool before next: prev <-> pool <-> next */ @@ -1084,7 +1543,7 @@ pymalloc_free(void *ctx, void *p) /* Link the pool to freepools. This is a singly-linked * list, and pool->prevpool isn't used there. */ - ao = &_PyRuntime.mem.arenas[pool->arenaindex]; + ao = &arenas[pool->arenaindex]; pool->nextpool = ao->freepools; ao->freepools = pool; nf = ++ao->nfreepools; @@ -1113,9 +1572,9 @@ pymalloc_free(void *ctx, void *p) * usable_arenas pointer. */ if (ao->prevarena == NULL) { - _PyRuntime.mem.usable_arenas = ao->nextarena; - assert(_PyRuntime.mem.usable_arenas == NULL || - _PyRuntime.mem.usable_arenas->address != 0); + usable_arenas = ao->nextarena; + assert(usable_arenas == NULL || + usable_arenas->address != 0); } else { assert(ao->prevarena->nextarena == ao); @@ -1131,14 +1590,14 @@ pymalloc_free(void *ctx, void *p) /* Record that this arena_object slot is * available to be reused. */ - ao->nextarena = _PyRuntime.mem.unused_arena_objects; - _PyRuntime.mem.unused_arena_objects = ao; + ao->nextarena = unused_arena_objects; + unused_arena_objects = ao; /* Free the entire arena. */ - _PyRuntime.obj.allocator_arenas.free(_PyRuntime.obj.allocator_arenas.ctx, + _PyObject_Arena.free(_PyObject_Arena.ctx, (void *)ao->address, ARENA_SIZE); ao->address = 0; /* mark unassociated */ - --_PyRuntime.mem.narenas_currently_allocated; + --narenas_currently_allocated; goto success; } @@ -1149,12 +1608,12 @@ pymalloc_free(void *ctx, void *p) * ao->nfreepools was 0 before, ao isn't * currently on the usable_arenas list. */ - ao->nextarena = _PyRuntime.mem.usable_arenas; + ao->nextarena = usable_arenas; ao->prevarena = NULL; - if (_PyRuntime.mem.usable_arenas) - _PyRuntime.mem.usable_arenas->prevarena = ao; - _PyRuntime.mem.usable_arenas = ao; - assert(_PyRuntime.mem.usable_arenas->address != 0); + if (usable_arenas) + usable_arenas->prevarena = ao; + usable_arenas = ao; + assert(usable_arenas->address != 0); goto success; } @@ -1183,8 +1642,8 @@ pymalloc_free(void *ctx, void *p) } else { /* ao is at the head of the list */ - assert(_PyRuntime.mem.usable_arenas == ao); - _PyRuntime.mem.usable_arenas = ao->nextarena; + assert(usable_arenas == ao); + usable_arenas = ao->nextarena; } ao->nextarena->prevarena = ao->prevarena; @@ -1209,7 +1668,7 @@ pymalloc_free(void *ctx, void *p) assert(ao->nextarena == NULL || nf <= ao->nextarena->nfreepools); assert(ao->prevarena == NULL || nf > ao->prevarena->nfreepools); assert(ao->nextarena == NULL || ao->nextarena->prevarena == ao); - assert((_PyRuntime.mem.usable_arenas == ao && ao->prevarena == NULL) + assert((usable_arenas == ao && ao->prevarena == NULL) || ao->prevarena->nextarena == ao); goto success; @@ -1228,7 +1687,7 @@ _PyObject_Free(void *ctx, void *p) return; } - _PyRuntime.mem.num_allocated_blocks--; + _Py_AllocatedBlocks--; if (!pymalloc_free(ctx, p)) { /* pymalloc didn't allocate this address */ PyMem_RawFree(p); @@ -1353,13 +1812,15 @@ _Py_GetAllocatedBlocks(void) #define DEADBYTE 0xDB /* dead (newly freed) memory */ #define FORBIDDENBYTE 0xFB /* untouchable bytes at each end of a block */ +static size_t serialno = 0; /* incremented on each debug {m,re}alloc */ + /* serialno is always incremented via calling this routine. The point is * to supply a single place to set a breakpoint. */ static void bumpserialno(void) { - ++_PyRuntime.mem.serialno; + ++serialno; } #define SST SIZEOF_SIZE_T @@ -1466,7 +1927,7 @@ _PyMem_DebugRawAlloc(int use_calloc, void *ctx, size_t nbytes) /* at tail, write pad (SST bytes) and serialno (SST bytes) */ tail = data + nbytes; memset(tail, FORBIDDENBYTE, SST); - write_size_t(tail + SST, _PyRuntime.mem.serialno); + write_size_t(tail + SST, serialno); return data; } @@ -1526,7 +1987,7 @@ _PyMem_DebugRawRealloc(void *ctx, void *p, size_t nbytes) uint8_t *tail; /* data + nbytes == pointer to tail pad bytes */ size_t total; /* 2 * SST + nbytes + 2 * SST */ size_t original_nbytes; - size_t serialno; + size_t block_serialno; #define ERASED_SIZE 64 uint8_t save[2*ERASED_SIZE]; /* A copy of erased bytes. */ @@ -1542,7 +2003,7 @@ _PyMem_DebugRawRealloc(void *ctx, void *p, size_t nbytes) total = nbytes + 4*SST; tail = data + original_nbytes; - serialno = read_size_t(tail + SST); + block_serialno = read_size_t(tail + SST); /* Mark the header, the trailer, ERASED_SIZE bytes at the begin and ERASED_SIZE bytes at the end as dead and save the copy of erased bytes. */ @@ -1565,7 +2026,7 @@ _PyMem_DebugRawRealloc(void *ctx, void *p, size_t nbytes) else { head = r; bumpserialno(); - serialno = _PyRuntime.mem.serialno; + block_serialno = serialno; } write_size_t(head, nbytes); @@ -1575,7 +2036,7 @@ _PyMem_DebugRawRealloc(void *ctx, void *p, size_t nbytes) tail = data + nbytes; memset(tail, FORBIDDENBYTE, SST); - write_size_t(tail + SST, serialno); + write_size_t(tail + SST, block_serialno); /* Restore saved bytes. */ if (original_nbytes <= sizeof(save)) { @@ -1923,16 +2384,16 @@ _PyObject_DebugMallocStats(FILE *out) * to march over all the arenas. If we're lucky, most of the memory * will be living in full pools -- would be a shame to miss them. */ - for (i = 0; i < _PyRuntime.mem.maxarenas; ++i) { + for (i = 0; i < maxarenas; ++i) { uint j; - uintptr_t base = _PyRuntime.mem.arenas[i].address; + uintptr_t base = arenas[i].address; /* Skip arenas which are not allocated. */ - if (_PyRuntime.mem.arenas[i].address == (uintptr_t)NULL) + if (arenas[i].address == (uintptr_t)NULL) continue; narenas += 1; - numfreepools += _PyRuntime.mem.arenas[i].nfreepools; + numfreepools += arenas[i].nfreepools; /* round up to pool alignment */ if (base & (uintptr_t)POOL_SIZE_MASK) { @@ -1942,8 +2403,8 @@ _PyObject_DebugMallocStats(FILE *out) } /* visit every pool in the arena */ - assert(base <= (uintptr_t) _PyRuntime.mem.arenas[i].pool_address); - for (j = 0; base < (uintptr_t) _PyRuntime.mem.arenas[i].pool_address; + assert(base <= (uintptr_t) arenas[i].pool_address); + for (j = 0; base < (uintptr_t) arenas[i].pool_address; ++j, base += POOL_SIZE) { poolp p = (poolp)base; const uint sz = p->szidx; @@ -1952,7 +2413,7 @@ _PyObject_DebugMallocStats(FILE *out) if (p->ref.count == 0) { /* currently unused */ #ifdef Py_DEBUG - assert(pool_is_in_list(p, _PyRuntime.mem.arenas[i].freepools)); + assert(pool_is_in_list(p, arenas[i].freepools)); #endif continue; } @@ -1962,11 +2423,11 @@ _PyObject_DebugMallocStats(FILE *out) numfreeblocks[sz] += freeblocks; #ifdef Py_DEBUG if (freeblocks > 0) - assert(pool_is_in_list(p, _PyRuntime.mem.usedpools[sz + sz])); + assert(pool_is_in_list(p, usedpools[sz + sz])); #endif } } - assert(narenas == _PyRuntime.mem.narenas_currently_allocated); + assert(narenas == narenas_currently_allocated); fputc('\n', out); fputs("class size num pools blocks in use avail blocks\n" @@ -1994,10 +2455,10 @@ _PyObject_DebugMallocStats(FILE *out) } fputc('\n', out); if (_PyMem_DebugEnabled()) - (void)printone(out, "# times object malloc called", _PyRuntime.mem.serialno); - (void)printone(out, "# arenas allocated total", _PyRuntime.mem.ntimes_arena_allocated); - (void)printone(out, "# arenas reclaimed", _PyRuntime.mem.ntimes_arena_allocated - narenas); - (void)printone(out, "# arenas highwater mark", _PyRuntime.mem.narenas_highwater); + (void)printone(out, "# times object malloc called", serialno); + (void)printone(out, "# arenas allocated total", ntimes_arena_allocated); + (void)printone(out, "# arenas reclaimed", ntimes_arena_allocated - narenas); + (void)printone(out, "# arenas highwater mark", narenas_highwater); (void)printone(out, "# arenas allocated current", narenas); PyOS_snprintf(buf, sizeof(buf), diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 246de78463d95b..3793cbda8829d8 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -116,7 +116,6 @@ - diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index b37e9930e3146c..1d33c6e2cc2802 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -141,9 +141,6 @@ Include - - Include - Include @@ -1031,4 +1028,4 @@ Resource Files - \ No newline at end of file + diff --git a/Parser/pgenmain.c b/Parser/pgenmain.c index dbdf13440afe3d..20ef8a71f710bc 100644 --- a/Parser/pgenmain.c +++ b/Parser/pgenmain.c @@ -60,8 +60,6 @@ main(int argc, char **argv) filename = argv[1]; graminit_h = argv[2]; graminit_c = argv[3]; - _PyObject_Initialize(&_PyRuntime.obj); - _PyMem_Initialize(&_PyRuntime.mem); g = getgrammar(filename); fp = fopen(graminit_c, "w"); if (fp == NULL) { diff --git a/Programs/_testembed.c b/Programs/_testembed.c index e68e68de327b12..52a0b5124a37f0 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -125,6 +125,28 @@ static int test_forced_io_encoding(void) return 0; } + +/********************************************************* + * Test parts of the C-API that work before initialization + *********************************************************/ + +static int test_pre_initialization_api(void) +{ + wchar_t *program = Py_DecodeLocale("spam", NULL); + if (program == NULL) { + fprintf(stderr, "Fatal error: cannot decode program name\n"); + return 1; + } + Py_SetProgramName(program); + + Py_Initialize(); + Py_Finalize(); + + PyMem_RawFree(program); + return 0; +} + + /* ********************************************************* * List of test cases and the function that implements it. * @@ -146,6 +168,7 @@ struct TestCase static struct TestCase TestCases[] = { { "forced_io_encoding", test_forced_io_encoding }, { "repeated_init_and_subinterpreters", test_repeated_init_and_subinterpreters }, + { "pre_initialization_api", test_pre_initialization_api }, { NULL, NULL } }; diff --git a/Python/pystate.c b/Python/pystate.c index f6fbb4d041ea4b..ecf921d0c25c53 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -40,8 +40,6 @@ _PyRuntimeState_Init(_PyRuntimeState *runtime) { memset(runtime, 0, sizeof(*runtime)); - _PyObject_Initialize(&runtime->obj); - _PyMem_Initialize(&runtime->mem); _PyGC_Initialize(&runtime->gc); _PyEval_Initialize(&runtime->ceval); From 19fb134185ce155bc53f517116fca73093ba55e9 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Fri, 24 Nov 2017 18:11:18 +0300 Subject: [PATCH 29/75] bpo-12239: Make GetProperty() return None for VT_EMPTY (GH-4539) The previous behavior was to raise an exception NotImplementedError: result of type 0 when the value of the property is VT_EMPTY. --- Lib/test/test_msilib.py | 7 +++++++ .../next/Library/2017-11-24-14-07-55.bpo-12239.Nj3A0x.rst | 2 ++ PC/_msi.c | 2 ++ 3 files changed, 11 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2017-11-24-14-07-55.bpo-12239.Nj3A0x.rst diff --git a/Lib/test/test_msilib.py b/Lib/test/test_msilib.py index 4093134195a26e..6d89ca4b77321e 100644 --- a/Lib/test/test_msilib.py +++ b/Lib/test/test_msilib.py @@ -53,6 +53,13 @@ def test_database_create_failed(self): msilib.OpenDatabase(db_path, msilib.MSIDBOPEN_CREATE) self.assertEqual(str(cm.exception), 'create failed') + def test_get_property_vt_empty(self): + db, db_path = init_database() + summary = db.GetSummaryInformation(0) + self.assertIsNone(summary.GetProperty(msilib.PID_SECURITY)) + db.Close() + self.addCleanup(unlink, db_path) + class Test_make_id(unittest.TestCase): #http://msdn.microsoft.com/en-us/library/aa369212(v=vs.85).aspx diff --git a/Misc/NEWS.d/next/Library/2017-11-24-14-07-55.bpo-12239.Nj3A0x.rst b/Misc/NEWS.d/next/Library/2017-11-24-14-07-55.bpo-12239.Nj3A0x.rst new file mode 100644 index 00000000000000..20d94779e4453a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-11-24-14-07-55.bpo-12239.Nj3A0x.rst @@ -0,0 +1,2 @@ +Make :meth:`msilib.SummaryInformation.GetProperty` return ``None`` when the +value of property is ``VT_EMPTY``. Initial patch by Mark Mc Mahon. diff --git a/PC/_msi.c b/PC/_msi.c index 8cc1fd59dd1926..000d81f139f354 100644 --- a/PC/_msi.c +++ b/PC/_msi.c @@ -575,6 +575,8 @@ summary_getproperty(msiobj* si, PyObject *args) if (sval != sbuf) free(sval); return result; + case VT_EMPTY: + Py_RETURN_NONE; } PyErr_Format(PyExc_NotImplementedError, "result of type %d", type); return NULL; From 0f86cd38f4a38f25a4aed3759a654a4b7fa49031 Mon Sep 17 00:00:00 2001 From: xdegaye Date: Fri, 24 Nov 2017 17:35:55 +0100 Subject: [PATCH 30/75] bpo-28684: asyncio tests handle PermissionError raised on binding unix sockets (GH-4503) The test.support.skip_unless_bind_unix_socket() decorator is used to skip asyncio tests that fail because the platform lacks a functional bind() function for unix domain sockets (as it is the case for non root users on the recent Android versions that run now SELinux in enforcing mode). --- Lib/test/support/__init__.py | 23 +++++++++++++++++++ Lib/test/test_asyncio/test_events.py | 15 ++++++------ Lib/test/test_asyncio/test_streams.py | 9 ++++---- Lib/test/test_asyncio/test_unix_events.py | 4 ++++ .../2017-11-22-12-54-46.bpo-28684.NLiDKZ.rst | 5 ++++ 5 files changed, 45 insertions(+), 11 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2017-11-22-12-54-46.bpo-28684.NLiDKZ.rst diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index b7cbdc6ab30542..e8648964d1e29e 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -88,6 +88,7 @@ "requires_IEEE_754", "skip_unless_xattr", "requires_zlib", "anticipate_failure", "load_package_tests", "detect_api_mismatch", "check__all__", "requires_android_level", "requires_multiprocessing_queue", + "skip_unless_bind_unix_socket", # sys "is_jython", "is_android", "check_impl_detail", "unix_shell", "setswitchinterval", @@ -2432,6 +2433,28 @@ def skip_unless_xattr(test): msg = "no non-broken extended attribute support" return test if ok else unittest.skip(msg)(test) +_bind_nix_socket_error = None +def skip_unless_bind_unix_socket(test): + """Decorator for tests requiring a functional bind() for unix sockets.""" + if not hasattr(socket, 'AF_UNIX'): + return unittest.skip('No UNIX Sockets')(test) + global _bind_nix_socket_error + if _bind_nix_socket_error is None: + path = TESTFN + "can_bind_unix_socket" + with socket.socket(socket.AF_UNIX) as sock: + try: + sock.bind(path) + _bind_nix_socket_error = False + except OSError as e: + _bind_nix_socket_error = e + finally: + unlink(path) + if _bind_nix_socket_error: + msg = 'Requires a functional unix bind(): %s' % _bind_nix_socket_error + return unittest.skip(msg)(test) + else: + return test + def fs_is_case_insensitive(directory): """Detects if the file system for the specified directory is case-insensitive.""" diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py index 1a8bc13429648f..78b30b9b6c3224 100644 --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -22,6 +22,7 @@ import unittest from unittest import mock import weakref +from test import support if sys.platform != 'win32': import tty @@ -470,7 +471,7 @@ def test_sock_client_ops(self): sock = socket.socket() self._basetest_sock_recv_into(httpd, sock) - @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'No UNIX Sockets') + @support.skip_unless_bind_unix_socket def test_unix_sock_client_ops(self): with test_utils.run_test_unix_server() as httpd: sock = socket.socket(socket.AF_UNIX) @@ -606,7 +607,7 @@ def test_create_connection(self): lambda: MyProto(loop=self.loop), *httpd.address) self._basetest_create_connection(conn_fut) - @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'No UNIX Sockets') + @support.skip_unless_bind_unix_socket def test_create_unix_connection(self): # Issue #20682: On Mac OS X Tiger, getsockname() returns a # zero-length address for UNIX socket. @@ -736,8 +737,8 @@ def test_create_ssl_connection(self): self._test_create_ssl_connection(httpd, create_connection, peername=httpd.address) + @support.skip_unless_bind_unix_socket @unittest.skipIf(ssl is None, 'No ssl module') - @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'No UNIX Sockets') def test_create_ssl_unix_connection(self): # Issue #20682: On Mac OS X Tiger, getsockname() returns a # zero-length address for UNIX socket. @@ -961,7 +962,7 @@ def _make_unix_server(self, factory, **kwargs): return server, path - @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'No UNIX Sockets') + @support.skip_unless_bind_unix_socket def test_create_unix_server(self): proto = MyProto(loop=self.loop) server, path = self._make_unix_server(lambda: proto) @@ -1053,8 +1054,8 @@ def test_create_server_ssl(self): # stop serving server.close() + @support.skip_unless_bind_unix_socket @unittest.skipIf(ssl is None, 'No ssl module') - @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'No UNIX Sockets') def test_create_unix_server_ssl(self): proto = MyProto(loop=self.loop) server, path = self._make_ssl_unix_server( @@ -1113,8 +1114,8 @@ def test_create_server_ssl_verify_failed(self): self.assertIsNone(proto.transport) server.close() + @support.skip_unless_bind_unix_socket @unittest.skipIf(ssl is None, 'No ssl module') - @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'No UNIX Sockets') def test_create_unix_server_ssl_verify_failed(self): proto = MyProto(loop=self.loop) server, path = self._make_ssl_unix_server( @@ -1171,8 +1172,8 @@ def test_create_server_ssl_match_failed(self): proto.transport.close() server.close() + @support.skip_unless_bind_unix_socket @unittest.skipIf(ssl is None, 'No ssl module') - @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'No UNIX Sockets') def test_create_unix_server_ssl_verified(self): proto = MyProto(loop=self.loop) server, path = self._make_ssl_unix_server( diff --git a/Lib/test/test_asyncio/test_streams.py b/Lib/test/test_asyncio/test_streams.py index 6d16d2007967d3..a1e5bd7fab6c8e 100644 --- a/Lib/test/test_asyncio/test_streams.py +++ b/Lib/test/test_asyncio/test_streams.py @@ -9,6 +9,7 @@ import threading import unittest from unittest import mock +from test import support try: import ssl except ImportError: @@ -57,7 +58,7 @@ def test_open_connection(self): loop=self.loop) self._basetest_open_connection(conn_fut) - @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'No UNIX Sockets') + @support.skip_unless_bind_unix_socket def test_open_unix_connection(self): with test_utils.run_test_unix_server() as httpd: conn_fut = asyncio.open_unix_connection(httpd.address, @@ -86,8 +87,8 @@ def test_open_connection_no_loop_ssl(self): self._basetest_open_connection_no_loop_ssl(conn_fut) + @support.skip_unless_bind_unix_socket @unittest.skipIf(ssl is None, 'No ssl module') - @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'No UNIX Sockets') def test_open_unix_connection_no_loop_ssl(self): with test_utils.run_test_unix_server(use_ssl=True) as httpd: conn_fut = asyncio.open_unix_connection( @@ -113,7 +114,7 @@ def test_open_connection_error(self): loop=self.loop) self._basetest_open_connection_error(conn_fut) - @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'No UNIX Sockets') + @support.skip_unless_bind_unix_socket def test_open_unix_connection_error(self): with test_utils.run_test_unix_server() as httpd: conn_fut = asyncio.open_unix_connection(httpd.address, @@ -634,7 +635,7 @@ def client(addr): server.stop() self.assertEqual(msg, b"hello world!\n") - @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'No UNIX Sockets') + @support.skip_unless_bind_unix_socket def test_start_unix_server(self): class MyServer: diff --git a/Lib/test/test_asyncio/test_unix_events.py b/Lib/test/test_asyncio/test_unix_events.py index fe758ba5e13529..04284fa57d1962 100644 --- a/Lib/test/test_asyncio/test_unix_events.py +++ b/Lib/test/test_asyncio/test_unix_events.py @@ -13,6 +13,7 @@ import threading import unittest from unittest import mock +from test import support if sys.platform == 'win32': raise unittest.SkipTest('UNIX only') @@ -239,6 +240,7 @@ def setUp(self): self.loop = asyncio.SelectorEventLoop() self.set_event_loop(self.loop) + @support.skip_unless_bind_unix_socket def test_create_unix_server_existing_path_sock(self): with test_utils.unix_socket_path() as path: sock = socket.socket(socket.AF_UNIX) @@ -251,6 +253,7 @@ def test_create_unix_server_existing_path_sock(self): srv.close() self.loop.run_until_complete(srv.wait_closed()) + @support.skip_unless_bind_unix_socket def test_create_unix_server_pathlib(self): with test_utils.unix_socket_path() as path: path = pathlib.Path(path) @@ -308,6 +311,7 @@ def test_create_unix_server_path_dgram(self): @unittest.skipUnless(hasattr(socket, 'SOCK_NONBLOCK'), 'no socket.SOCK_NONBLOCK (linux only)') + @support.skip_unless_bind_unix_socket def test_create_unix_server_path_stream_bittype(self): sock = socket.socket( socket.AF_UNIX, socket.SOCK_STREAM | socket.SOCK_NONBLOCK) diff --git a/Misc/NEWS.d/next/Library/2017-11-22-12-54-46.bpo-28684.NLiDKZ.rst b/Misc/NEWS.d/next/Library/2017-11-22-12-54-46.bpo-28684.NLiDKZ.rst new file mode 100644 index 00000000000000..9d8e4da822f377 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-11-22-12-54-46.bpo-28684.NLiDKZ.rst @@ -0,0 +1,5 @@ +The new test.support.skip_unless_bind_unix_socket() decorator is used here to +skip asyncio tests that fail because the platform lacks a functional bind() +function for unix domain sockets (as it is the case for non root users on the +recent Android versions that run now SELinux in enforcing mode). + From 78a5722ae950b80a4b3d13377957f3932195aef3 Mon Sep 17 00:00:00 2001 From: Will White Date: Fri, 24 Nov 2017 17:28:12 +0000 Subject: [PATCH 31/75] Improve the String tutorial docs (GH-4541) The paragraph that contains example of string literal concatenation was placed after the section about concatenation using the '+' sign. Moved the paragraph to the appropriate section. --- Doc/tutorial/introduction.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Doc/tutorial/introduction.rst b/Doc/tutorial/introduction.rst index 2fa894a533f3a8..7176d819425094 100644 --- a/Doc/tutorial/introduction.rst +++ b/Doc/tutorial/introduction.rst @@ -212,6 +212,13 @@ to each other are automatically concatenated. :: >>> 'Py' 'thon' 'Python' +This feature is particularly useful when you want to break long strings:: + + >>> text = ('Put several strings within parentheses ' + ... 'to have them joined together.') + >>> text + 'Put several strings within parentheses to have them joined together.' + This only works with two literals though, not with variables or expressions:: >>> prefix = 'Py' @@ -227,13 +234,6 @@ If you want to concatenate variables or a variable and a literal, use ``+``:: >>> prefix + 'thon' 'Python' -This feature is particularly useful when you want to break long strings:: - - >>> text = ('Put several strings within parentheses ' - ... 'to have them joined together.') - >>> text - 'Put several strings within parentheses to have them joined together.' - Strings can be *indexed* (subscripted), with the first character having index 0. There is no separate character type; a character is simply a string of size one:: From 5742f674f7561dc9a1fc66d650e18e31b941183b Mon Sep 17 00:00:00 2001 From: xdegaye Date: Fri, 24 Nov 2017 18:56:22 +0100 Subject: [PATCH 32/75] bpo-28684: Remove useless import added by the previous commit (GH-4547) --- Lib/test/test_asyncio/test_events.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py index 78b30b9b6c3224..a1079128d0d726 100644 --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -22,7 +22,6 @@ import unittest from unittest import mock import weakref -from test import support if sys.platform != 'win32': import tty From da9c8c36aeb60ad8f7748a735c372bf993d2e4f3 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 24 Nov 2017 22:06:38 +0100 Subject: [PATCH 33/75] bpo-32125: Remove Py_UseClassExceptionsFlag flag (#4544) This flag was deprecated and wasn't used anymore since Python 2.0. --- Include/pydebug.h | 1 - Misc/NEWS.d/next/C API/2017-11-24-21-25-43.bpo-32125.K8zWgn.rst | 2 ++ Python/pylifecycle.c | 1 - Tools/c-globals/ignored-globals.txt | 1 - 4 files changed, 2 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2017-11-24-21-25-43.bpo-32125.K8zWgn.rst diff --git a/Include/pydebug.h b/Include/pydebug.h index d3b95966aa604e..bd4aafe3b49f83 100644 --- a/Include/pydebug.h +++ b/Include/pydebug.h @@ -15,7 +15,6 @@ PyAPI_DATA(int) Py_InspectFlag; PyAPI_DATA(int) Py_OptimizeFlag; PyAPI_DATA(int) Py_NoSiteFlag; PyAPI_DATA(int) Py_BytesWarningFlag; -PyAPI_DATA(int) Py_UseClassExceptionsFlag; PyAPI_DATA(int) Py_FrozenFlag; PyAPI_DATA(int) Py_IgnoreEnvironmentFlag; PyAPI_DATA(int) Py_DontWriteBytecodeFlag; diff --git a/Misc/NEWS.d/next/C API/2017-11-24-21-25-43.bpo-32125.K8zWgn.rst b/Misc/NEWS.d/next/C API/2017-11-24-21-25-43.bpo-32125.K8zWgn.rst new file mode 100644 index 00000000000000..d71c66415d3e00 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2017-11-24-21-25-43.bpo-32125.K8zWgn.rst @@ -0,0 +1,2 @@ +The ``Py_UseClassExceptionsFlag`` flag has been removed. It was deprecated +and wasn't used anymore since Python 2.0. diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 8d2ec4e91c3891..b079990c40f2bd 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -116,7 +116,6 @@ int Py_InspectFlag; /* Needed to determine whether to exit at SystemExit */ int Py_OptimizeFlag = 0; /* Needed by compile.c */ int Py_NoSiteFlag; /* Suppress 'import site' */ int Py_BytesWarningFlag; /* Warn on str(bytes) and str(buffer) */ -int Py_UseClassExceptionsFlag = 1; /* Needed by bltinmodule.c: deprecated */ int Py_FrozenFlag; /* Needed by getpath.c */ int Py_IgnoreEnvironmentFlag; /* e.g. PYTHONPATH, PYTHONHOME */ int Py_DontWriteBytecodeFlag; /* Suppress writing bytecode files (*.pyc) */ diff --git a/Tools/c-globals/ignored-globals.txt b/Tools/c-globals/ignored-globals.txt index 7b5add865c166c..ce6d1d805147b6 100644 --- a/Tools/c-globals/ignored-globals.txt +++ b/Tools/c-globals/ignored-globals.txt @@ -393,7 +393,6 @@ Py_NoUserSiteDirectory Py_OptimizeFlag Py_QuietFlag Py_UnbufferedStdioFlag -Py_UseClassExceptionsFlag Py_VerboseFlag From 84c4b1938fade2b425ac906730beabd413de094d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 24 Nov 2017 22:30:27 +0100 Subject: [PATCH 34/75] bpo-32124: Document C functions safe before init (#4540) Explicitly document C functions and C variables that can be set before Py_Initialize(). --- Doc/c-api/init.rst | 215 +++++++++++++++++++++++++++++++++++++++++- Doc/using/cmdline.rst | 2 +- Misc/python.man | 2 +- 3 files changed, 214 insertions(+), 5 deletions(-) diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index dc1939db17e4c3..2f77bb359d24e5 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -7,6 +7,213 @@ Initialization, Finalization, and Threads ***************************************** +.. _pre-init-safe: + +Before Python Initialization +============================ + +In an application embedding Python, the :c:func:`Py_Initialize` function must +be called before using any other Python/C API functions; with the exception of +a few functions and the :ref:`global configuration variables +`. + +The following functions can be safely called before Python is initialized: + +* Configuration functions: + + * :c:func:`PyImport_AppendInittab` + * :c:func:`PyImport_ExtendInittab` + * :c:func:`PyInitFrozenExtensions` + * :c:func:`PyMem_SetAllocator` + * :c:func:`PyMem_SetupDebugHooks` + * :c:func:`PyObject_SetArenaAllocator` + * :c:func:`Py_SetPath` + * :c:func:`Py_SetProgramName` + * :c:func:`Py_SetPythonHome` + * :c:func:`Py_SetStandardStreamEncoding` + +* Informative functions: + + * :c:func:`PyMem_GetAllocator` + * :c:func:`PyObject_GetArenaAllocator` + * :c:func:`Py_GetBuildInfo` + * :c:func:`Py_GetCompiler` + * :c:func:`Py_GetCopyright` + * :c:func:`Py_GetPlatform` + * :c:func:`Py_GetProgramName` + * :c:func:`Py_GetVersion` + +* Utilities: + + * :c:func:`Py_DecodeLocale` + +* Memory allocators: + + * :c:func:`PyMem_RawMalloc` + * :c:func:`PyMem_RawRealloc` + * :c:func:`PyMem_RawCalloc` + * :c:func:`PyMem_RawFree` + +.. note:: + + The following functions **should not be called** before + :c:func:`Py_Initialize`: :c:func:`Py_EncodeLocale`, :c:func:`Py_GetPath`, + :c:func:`Py_GetPrefix`, :c:func:`Py_GetExecPrefix` and + :c:func:`Py_GetProgramFullPath` and :c:func:`Py_GetPythonHome`. + + +.. _global-conf-vars: + +Global configuration variables +============================== + +Python has variables for the global configuration to control different features +and options. By default, these flags are controlled by :ref:`command line +options `. + +When a flag is set by an option, the value of the flag is the number of times +that the option was set. For example, ``-b`` sets :c:data:`Py_BytesWarningFlag` +to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. + +.. c:var:: Py_BytesWarningFlag + + Issue a warning when comparing :class:`bytes` or :class:`bytearray` with + :class:`str` or :class:`bytes` with :class:`int`. Issue an error if greater + or equal to ``2``. + + Set by the :option:`-b` option. + +.. c:var:: Py_DebugFlag + + Turn on parser debugging output (for expert only, depending on compilation + options). + + Set by the :option:`-d` option and the :envvar:`PYTHONDEBUG` environment + variable. + +.. c:var:: Py_DontWriteBytecodeFlag + + If set to non-zero, Python won't try to write ``.pyc`` files on the + import of source modules. + + Set by the :option:`-B` option and the :envvar:`PYTHONDONTWRITEBYTECODE` + environment variable. + +.. c:var:: Py_FrozenFlag + + Suppress error messages when calculating the module search path in + :c:func:`Py_GetPath`. + + Private flag used by ``_freeze_importlib`` and ``frozenmain`` programs. + +.. c:var:: Py_HashRandomizationFlag + + Set to ``1`` if the :envvar:`PYTHONHASHSEED` environment variable is set to + a non-empty string. + + If the flag is non-zero, read the :envvar:`PYTHONHASHSEED` environment + variable to initialize the secret hash seed. + +.. c:var:: Py_IgnoreEnvironmentFlag + + Ignore all :envvar:`PYTHON*` environment variables, e.g. + :envvar:`PYTHONPATH` and :envvar:`PYTHONHOME`, that might be set. + + Set by the :option:`-E` and :option:`-I` options. + +.. c:var:: Py_InspectFlag + + When a script is passed as first argument or the :option:`-c` option is used, + enter interactive mode after executing the script or the command, even when + :data:`sys.stdin` does not appear to be a terminal. + + Set by the :option:`-i` option and the :envvar:`PYTHONINSPECT` environment + variable. + +.. c:var:: Py_InteractiveFlag + + Set by the :option:`-i` option. + +.. c:var:: Py_IsolatedFlag + + Run Python in isolated mode. In isolated mode :data:`sys.path` contains + neither the script's directory nor the user's site-packages directory. + + Set by the :option:`-I` option. + + .. versionadded:: 3.4 + +.. c:var:: Py_LegacyWindowsFSEncodingFlag + + If the flag is non-zero, use the ``mbcs`` encoding instead of the UTF-8 + encoding for the filesystem encoding. + + Set to ``1`` if the :envvar:`PYTHONLEGACYWINDOWSFSENCODING` environment + variable is set to a non-empty string. + + See :pep:`529` for more details. + + Availability: Windows. + +.. c:var:: Py_LegacyWindowsStdioFlag + + If the flag is non-zero, use :class:`io.FileIO` instead of + :class:`WindowsConsoleIO` for :mod:`sys` standard streams. + + Set to ``1`` if the :envvar:`PYTHONLEGACYWINDOWSSTDIO` environment + variable is set to a non-empty string. + + See :pep:`528` for more details. + + Availability: Windows. + +.. c:var:: Py_NoSiteFlag + + Disable the import of the module :mod:`site` and the site-dependent + manipulations of :data:`sys.path` that it entails. Also disable these + manipulations if :mod:`site` is explicitly imported later (call + :func:`site.main` if you want them to be triggered). + + Set by the :option:`-S` option. + +.. c:var:: Py_NoUserSiteDirectory + + Don't add the :data:`user site-packages directory ` to + :data:`sys.path`. + + Set by the :option:`-s` and :option:`-I` options, and the + :envvar:`PYTHONNOUSERSITE` environment variable. + +.. c:var:: Py_OptimizeFlag + + Set by the :option:`-O` option and the :envvar:`PYTHONOPTIMIZE` environment + variable. + +.. c:var:: Py_QuietFlag + + Don't display the copyright and version messages even in interactive mode. + + Set by the :option:`-q` option. + + .. versionadded:: 3.2 + +.. c:var:: Py_UnbufferedStdioFlag + + Force the stdout and stderr streams to be unbuffered. + + Set by the :option:`-u` option and the :envvar:`PYTHONUNBUFFERED` + environment variable. + +.. c:var:: Py_VerboseFlag + + Print a message each time a module is initialized, showing the place + (filename or built-in module) from which it is loaded. If greater or equal + to ``2``, print a message for each file that is checked for when + searching for a module. Also provides information on module cleanup at exit. + + Set by the :option:`-v` option and the :envvar:`PYTHONVERBOSE` environment + variable. + Initializing and finalizing the interpreter =========================================== @@ -27,9 +234,11 @@ Initializing and finalizing the interpreter single: PySys_SetArgvEx() single: Py_FinalizeEx() - Initialize the Python interpreter. In an application embedding Python, this - should be called before using any other Python/C API functions; with the - exception of :c:func:`Py_SetProgramName`, :c:func:`Py_SetPythonHome` and :c:func:`Py_SetPath`. This initializes + Initialize the Python interpreter. In an application embedding Python, + this should be called before using any other Python/C API functions; see + :ref:`Before Python Initialization ` for the few exceptions. + + This initializes the table of loaded modules (``sys.modules``), and creates the fundamental modules :mod:`builtins`, :mod:`__main__` and :mod:`sys`. It also initializes the module search path (``sys.path``). It does not set ``sys.argv``; use diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index fc557eebe4ac3e..d022e2cd2a0ace 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -212,7 +212,7 @@ Miscellaneous options .. cmdoption:: -d - Turn on parser debugging output (for wizards only, depending on compilation + Turn on parser debugging output (for expert only, depending on compilation options). See also :envvar:`PYTHONDEBUG`. diff --git a/Misc/python.man b/Misc/python.man index 9f71d69dfaf260..b4110295536606 100644 --- a/Misc/python.man +++ b/Misc/python.man @@ -124,7 +124,7 @@ This terminates the option list (following options are passed as arguments to the command). .TP .B \-d -Turn on parser debugging output (for wizards only, depending on +Turn on parser debugging output (for expert only, depending on compilation options). .TP .B \-E From 46972b7bc385ec2bdc7f567bbd22c9e56ffdf003 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 24 Nov 2017 22:55:40 +0100 Subject: [PATCH 35/75] bpo-32030: Add _PyMainInterpreterConfig_ReadEnv() (#4542) Py_GetPath() and Py_Main() now call _PyMainInterpreterConfig_ReadEnv() to share the same code to get environment variables. Changes: * Add _PyMainInterpreterConfig_ReadEnv() * Add _PyMainInterpreterConfig_Clear() * Add _PyMem_RawWcsdup() * _PyMainInterpreterConfig: rename pythonhome to home * Rename _Py_ReadMainInterpreterConfig() to _PyMainInterpreterConfig_Read() * Use _Py_INIT_USER_ERR(), instead of _Py_INIT_ERR(), for decoding errors: the user is able to fix the issue, it's not a bug in Python. Same change was made in _Py_INIT_NO_MEMORY(). * Remove _Py_GetPythonHomeWithConfig() --- Include/pylifecycle.h | 13 +++--- Include/pymem.h | 6 +++ Include/pystate.h | 9 ++-- Modules/getpath.c | 70 +++++++++++++++---------------- Modules/main.c | 97 +++++++++++++++++++++++-------------------- Objects/obmalloc.c | 18 ++++++++ PC/getpathp.c | 49 ++++++++++++---------- Python/pylifecycle.c | 45 +++++++++----------- 8 files changed, 165 insertions(+), 142 deletions(-) diff --git a/Include/pylifecycle.h b/Include/pylifecycle.h index efdc58eed8ebf2..ff915323386594 100644 --- a/Include/pylifecycle.h +++ b/Include/pylifecycle.h @@ -30,7 +30,7 @@ typedef struct { Don't abort() the process on such error. */ #define _Py_INIT_USER_ERR(MSG) \ (_PyInitError){.prefix = _Py_INIT_GET_FUNC(), .msg = (MSG), .user_err = 1} -#define _Py_INIT_NO_MEMORY() _Py_INIT_ERR("memory allocation failed") +#define _Py_INIT_NO_MEMORY() _Py_INIT_USER_ERR("memory allocation failed") #define _Py_INIT_FAILED(err) \ (err.msg != NULL) @@ -42,11 +42,6 @@ PyAPI_FUNC(wchar_t *) Py_GetProgramName(void); PyAPI_FUNC(void) Py_SetPythonHome(wchar_t *); PyAPI_FUNC(wchar_t *) Py_GetPythonHome(void); -#ifdef Py_BUILD_CORE -PyAPI_FUNC(_PyInitError) _Py_GetPythonHomeWithConfig( - const _PyMainInterpreterConfig *config, - wchar_t **home); -#endif #ifndef Py_LIMITED_API /* Only used by applications that embed the interpreter and need to @@ -58,7 +53,11 @@ PyAPI_FUNC(int) Py_SetStandardStreamEncoding(const char *encoding, /* PEP 432 Multi-phase initialization API (Private while provisional!) */ PyAPI_FUNC(_PyInitError) _Py_InitializeCore(const _PyCoreConfig *); PyAPI_FUNC(int) _Py_IsCoreInitialized(void); -PyAPI_FUNC(_PyInitError) _Py_ReadMainInterpreterConfig(_PyMainInterpreterConfig *); + +PyAPI_FUNC(_PyInitError) _PyMainInterpreterConfig_Read(_PyMainInterpreterConfig *); +PyAPI_FUNC(_PyInitError) _PyMainInterpreterConfig_ReadEnv(_PyMainInterpreterConfig *); +PyAPI_FUNC(void) _PyMainInterpreterConfig_Clear(_PyMainInterpreterConfig *); + PyAPI_FUNC(_PyInitError) _Py_InitializeMainInterpreter(const _PyMainInterpreterConfig *); #endif diff --git a/Include/pymem.h b/Include/pymem.h index 928851a3d70e45..57a34cf90783b6 100644 --- a/Include/pymem.h +++ b/Include/pymem.h @@ -105,8 +105,14 @@ PyAPI_FUNC(void *) PyMem_Realloc(void *ptr, size_t new_size); PyAPI_FUNC(void) PyMem_Free(void *ptr); #ifndef Py_LIMITED_API +/* strdup() using PyMem_RawMalloc() */ PyAPI_FUNC(char *) _PyMem_RawStrdup(const char *str); + +/* strdup() using PyMem_Malloc() */ PyAPI_FUNC(char *) _PyMem_Strdup(const char *str); + +/* wcsdup() using PyMem_RawMalloc() */ +PyAPI_FUNC(wchar_t*) _PyMem_RawWcsdup(const wchar_t *str); #endif /* Macros. */ diff --git a/Include/pystate.h b/Include/pystate.h index ab6400cddc8c55..533851fc0283af 100644 --- a/Include/pystate.h +++ b/Include/pystate.h @@ -60,16 +60,17 @@ typedef struct { */ typedef struct { int install_signal_handlers; - wchar_t *module_search_path_env; /* PYTHONPATH environment variable */ - wchar_t *pythonhome; /* PYTHONHOME environment variable, - see also Py_SetPythonHome(). */ + /* PYTHONPATH environment variable */ + wchar_t *module_search_path_env; + /* PYTHONHOME environment variable, see also Py_SetPythonHome(). */ + wchar_t *home; } _PyMainInterpreterConfig; #define _PyMainInterpreterConfig_INIT \ (_PyMainInterpreterConfig){\ .install_signal_handlers = -1, \ .module_search_path_env = NULL, \ - .pythonhome = NULL} + .home = NULL} typedef struct _is { diff --git a/Modules/getpath.c b/Modules/getpath.c index 48bdd0f21daca7..db28eaf66df2bc 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -120,7 +120,6 @@ typedef struct { wchar_t *path_env; /* PATH environment variable */ wchar_t *home; /* PYTHONHOME environment variable */ wchar_t *module_search_path_env; /* PYTHONPATH environment variable */ - wchar_t *module_search_path_buffer; wchar_t *prog; /* Program name */ wchar_t *pythonpath; /* PYTHONPATH define */ @@ -886,9 +885,9 @@ calculate_module_search_path(PyCalculatePath *calculate, PyPathConfig *config) } -#define DECODE_FAILED(NAME, LEN) \ +#define DECODE_LOCALE_ERR(NAME, LEN) \ ((LEN) == (size_t)-2) \ - ? _Py_INIT_ERR("failed to decode " #NAME) \ + ? _Py_INIT_USER_ERR("failed to decode " #NAME) \ : _Py_INIT_NO_MEMORY() @@ -896,19 +895,15 @@ static _PyInitError calculate_init(PyCalculatePath *calculate, const _PyMainInterpreterConfig *main_config) { - _PyInitError err; - - err = _Py_GetPythonHomeWithConfig(main_config, &calculate->home); - if (_Py_INIT_FAILED(err)) { - return err; - } + calculate->home = main_config->home; + calculate->module_search_path_env = main_config->module_search_path_env; size_t len; char *path = getenv("PATH"); if (path) { calculate->path_env = Py_DecodeLocale(path, &len); if (!calculate->path_env) { - return DECODE_FAILED("PATH environment variable", len); + return DECODE_LOCALE_ERR("PATH environment variable", len); } } @@ -916,37 +911,19 @@ calculate_init(PyCalculatePath *calculate, calculate->pythonpath = Py_DecodeLocale(PYTHONPATH, &len); if (!calculate->pythonpath) { - return DECODE_FAILED("PYTHONPATH define", len); + return DECODE_LOCALE_ERR("PYTHONPATH define", len); } calculate->prefix = Py_DecodeLocale(PREFIX, &len); if (!calculate->prefix) { - return DECODE_FAILED("PREFIX define", len); + return DECODE_LOCALE_ERR("PREFIX define", len); } calculate->exec_prefix = Py_DecodeLocale(EXEC_PREFIX, &len); if (!calculate->prefix) { - return DECODE_FAILED("EXEC_PREFIX define", len); + return DECODE_LOCALE_ERR("EXEC_PREFIX define", len); } calculate->lib_python = Py_DecodeLocale("lib/python" VERSION, &len); if (!calculate->lib_python) { - return DECODE_FAILED("EXEC_PREFIX define", len); - } - - calculate->module_search_path_env = NULL; - if (main_config) { - if (main_config->module_search_path_env) { - calculate->module_search_path_env = main_config->module_search_path_env; - } - - } - else { - char *pythonpath = Py_GETENV("PYTHONPATH"); - if (pythonpath && pythonpath[0] != '\0') { - calculate->module_search_path_buffer = Py_DecodeLocale(pythonpath, &len); - if (!calculate->module_search_path_buffer) { - return DECODE_FAILED("PYTHONPATH environment variable", len); - } - calculate->module_search_path_env = calculate->module_search_path_buffer; - } + return DECODE_LOCALE_ERR("EXEC_PREFIX define", len); } return _Py_INIT_OK(); } @@ -960,7 +937,6 @@ calculate_free(PyCalculatePath *calculate) PyMem_RawFree(calculate->exec_prefix); PyMem_RawFree(calculate->lib_python); PyMem_RawFree(calculate->path_env); - PyMem_RawFree(calculate->module_search_path_buffer); } @@ -988,13 +964,24 @@ calculate_path_impl(PyCalculatePath *calculate, PyPathConfig *config) static void calculate_path(const _PyMainInterpreterConfig *main_config) { + _PyInitError err; PyCalculatePath calculate; memset(&calculate, 0, sizeof(calculate)); - _PyInitError err = calculate_init(&calculate, main_config); + _PyMainInterpreterConfig tmp_config; + int use_tmp = (main_config == NULL); + if (use_tmp) { + tmp_config = _PyMainInterpreterConfig_INIT; + err = _PyMainInterpreterConfig_ReadEnv(&tmp_config); + if (_Py_INIT_FAILED(err)) { + goto fatal_error; + } + main_config = &tmp_config; + } + + err = calculate_init(&calculate, main_config); if (_Py_INIT_FAILED(err)) { - calculate_free(&calculate); - _Py_FatalInitError(err); + goto fatal_error; } PyPathConfig new_path_config; @@ -1003,7 +990,18 @@ calculate_path(const _PyMainInterpreterConfig *main_config) calculate_path_impl(&calculate, &new_path_config); path_config = new_path_config; + if (use_tmp) { + _PyMainInterpreterConfig_Clear(&tmp_config); + } + calculate_free(&calculate); + return; + +fatal_error: + if (use_tmp) { + _PyMainInterpreterConfig_Clear(&tmp_config); + } calculate_free(&calculate); + _Py_FatalInitError(err); } diff --git a/Modules/main.c b/Modules/main.c index 349d8c3f3239ad..5b0c04938411e6 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -36,10 +36,16 @@ extern "C" { #endif +#define DECODE_LOCALE_ERR(NAME, LEN) \ + (((LEN) == -2) \ + ? _Py_INIT_USER_ERR("failed to decode " #NAME) \ + : _Py_INIT_NO_MEMORY()) + + #define SET_DECODE_ERROR(NAME, LEN) \ do { \ if ((LEN) == (size_t)-2) { \ - pymain->err = _Py_INIT_ERR("failed to decode " #NAME); \ + pymain->err = _Py_INIT_USER_ERR("failed to decode " #NAME); \ } \ else { \ pymain->err = _Py_INIT_NO_MEMORY(); \ @@ -450,7 +456,7 @@ pymain_free_impl(_PyMain *pymain) Py_CLEAR(pymain->main_importer_path); PyMem_RawFree(pymain->program_name); - PyMem_RawFree(pymain->config.module_search_path_env); + _PyMainInterpreterConfig_Clear(&pymain->config); #ifdef __INSURE__ /* Insure++ is a memory analysis tool that aids in discovering @@ -515,20 +521,11 @@ pymain_run_main_from_importer(_PyMain *pymain) static wchar_t* pymain_wstrdup(_PyMain *pymain, wchar_t *str) { - size_t len = wcslen(str) + 1; /* +1 for NUL character */ - if (len > (size_t)PY_SSIZE_T_MAX / sizeof(wchar_t)) { - pymain->err = _Py_INIT_NO_MEMORY(); - return NULL; - } - - size_t size = len * sizeof(wchar_t); - wchar_t *str2 = PyMem_RawMalloc(size); + wchar_t *str2 = _PyMem_RawWcsdup(str); if (str2 == NULL) { pymain->err = _Py_INIT_NO_MEMORY(); return NULL; } - - memcpy(str2, str, size); return str2; } @@ -955,7 +952,7 @@ pymain_init_main_interpreter(_PyMain *pymain) _PyInitError err; /* TODO: Print any exceptions raised by these operations */ - err = _Py_ReadMainInterpreterConfig(&pymain->config); + err = _PyMainInterpreterConfig_Read(&pymain->config); if (_Py_INIT_FAILED(err)) { pymain->err = err; return -1; @@ -1361,8 +1358,7 @@ pymain_set_flags_from_env(_PyMain *pymain) static int -pymain_get_env_var_dup(_PyMain *pymain, wchar_t **dest, - wchar_t *wname, char *name) +config_get_env_var_dup(wchar_t **dest, wchar_t *wname, char *name) { if (Py_IgnoreEnvironmentFlag) { *dest = NULL; @@ -1376,7 +1372,7 @@ pymain_get_env_var_dup(_PyMain *pymain, wchar_t **dest, return 0; } - wchar_t *copy = pymain_wstrdup(pymain, var); + wchar_t *copy = _PyMem_RawWcsdup(var); if (copy == NULL) { return -1; } @@ -1393,11 +1389,9 @@ pymain_get_env_var_dup(_PyMain *pymain, wchar_t **dest, wchar_t *wvar = Py_DecodeLocale(var, &len); if (!wvar) { if (len == (size_t)-2) { - /* don't set pymain->err */ return -2; } else { - pymain->err = _Py_INIT_NO_MEMORY(); return -1; } } @@ -1407,25 +1401,21 @@ pymain_get_env_var_dup(_PyMain *pymain, wchar_t **dest, } -static int -pymain_init_pythonpath(_PyMain *pymain) +static _PyInitError +config_init_pythonpath(_PyMainInterpreterConfig *config) { wchar_t *path; - int res = pymain_get_env_var_dup(pymain, &path, - L"PYTHONPATH", "PYTHONPATH"); + int res = config_get_env_var_dup(&path, L"PYTHONPATH", "PYTHONPATH"); if (res < 0) { - if (res == -2) { - SET_DECODE_ERROR("PYTHONPATH", (size_t)-2); - } - return -1; + return DECODE_LOCALE_ERR("PYTHONHOME", res); } - pymain->config.module_search_path_env = path; - return 0; + config->module_search_path_env = path; + return _Py_INIT_OK(); } -static int -pymain_init_pythonhome(_PyMain *pymain) +static _PyInitError +config_init_pythonhome(_PyMainInterpreterConfig *config) { wchar_t *home; @@ -1433,26 +1423,41 @@ pymain_init_pythonhome(_PyMain *pymain) if (home) { /* Py_SetPythonHome() has been called before Py_Main(), use its value */ - pymain->config.pythonhome = pymain_wstrdup(pymain, home); - if (pymain->config.pythonhome == NULL) { - return -1; + config->home = _PyMem_RawWcsdup(home); + if (config->home == NULL) { + return _Py_INIT_NO_MEMORY(); } - return 0; + return _Py_INIT_OK(); } - int res = pymain_get_env_var_dup(pymain, &home, - L"PYTHONHOME", "PYTHONHOME"); + int res = config_get_env_var_dup(&home, L"PYTHONHOME", "PYTHONHOME"); if (res < 0) { - if (res == -2) { - SET_DECODE_ERROR("PYTHONHOME", (size_t)-2); - } - return -1; + return DECODE_LOCALE_ERR("PYTHONHOME", res); } - pymain->config.pythonhome = home; - return 0; + config->home = home; + return _Py_INIT_OK(); } +_PyInitError +_PyMainInterpreterConfig_ReadEnv(_PyMainInterpreterConfig *config) +{ + _PyInitError err = config_init_pythonhome(config); + if (_Py_INIT_FAILED(err)) { + return err; + } + + err = config_init_pythonpath(config); + if (_Py_INIT_FAILED(err)) { + return err; + } + + return _Py_INIT_OK(); +} + + + + static int pymain_parse_envvars(_PyMain *pymain) { @@ -1474,10 +1479,10 @@ pymain_parse_envvars(_PyMain *pymain) return -1; } core_config->allocator = Py_GETENV("PYTHONMALLOC"); - if (pymain_init_pythonpath(pymain) < 0) { - return -1; - } - if (pymain_init_pythonhome(pymain) < 0) { + + _PyInitError err = _PyMainInterpreterConfig_ReadEnv(&pymain->config); + if (_Py_INIT_FAILED(pymain->err)) { + pymain->err = err; return -1; } diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index 96a451ee498a16..9bd97986300f3d 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -455,6 +455,24 @@ PyMem_Free(void *ptr) } +wchar_t* +_PyMem_RawWcsdup(const wchar_t *str) +{ + size_t len = wcslen(str); + if (len > (size_t)PY_SSIZE_T_MAX / sizeof(wchar_t) - 1) { + return NULL; + } + + size_t size = (len + 1) * sizeof(wchar_t); + wchar_t *str2 = PyMem_RawMalloc(size); + if (str2 == NULL) { + return NULL; + } + + memcpy(str2, str, size); + return str2; +} + char * _PyMem_RawStrdup(const char *str) { diff --git a/PC/getpathp.c b/PC/getpathp.c index e0cb9a25758496..5adf16d6f27384 100644 --- a/PC/getpathp.c +++ b/PC/getpathp.c @@ -689,26 +689,12 @@ read_pth_file(const wchar_t *path, wchar_t *prefix, int *isolated, int *nosite) } -static _PyInitError +static void calculate_init(PyCalculatePath *calculate, const _PyMainInterpreterConfig *main_config) { - _PyInitError err; - - err = _Py_GetPythonHomeWithConfig(main_config, &calculate->home); - if (_Py_INIT_FAILED(err)) { - return err; - } - - if (main_config) { - calculate->module_search_path_env = main_config->module_search_path_env; - } - else if (!Py_IgnoreEnvironmentFlag) { - wchar_t *path = _wgetenv(L"PYTHONPATH"); - if (path && *path != '\0') { - calculate->module_search_path_env = path; - } - } + calculate->home = main_config->home; + calculate->module_search_path_env = main_config->module_search_path_env; calculate->path_env = _wgetenv(L"PATH"); @@ -717,8 +703,6 @@ calculate_init(PyCalculatePath *calculate, prog = L"python"; } calculate->prog = prog; - - return _Py_INIT_OK(); } @@ -1020,22 +1004,41 @@ calculate_free(PyCalculatePath *calculate) static void calculate_path(const _PyMainInterpreterConfig *main_config) { + _PyInitError err; PyCalculatePath calculate; memset(&calculate, 0, sizeof(calculate)); - _PyInitError err = calculate_init(&calculate, main_config); - if (_Py_INIT_FAILED(err)) { - calculate_free(&calculate); - _Py_FatalInitError(err); + _PyMainInterpreterConfig tmp_config; + int use_tmp = (main_config == NULL); + if (use_tmp) { + tmp_config = _PyMainInterpreterConfig_INIT; + err = _PyMainInterpreterConfig_ReadEnv(&tmp_config); + if (_Py_INIT_FAILED(err)) { + goto fatal_error; + } + main_config = &tmp_config; } + calculate_init(&calculate, main_config); + PyPathConfig new_path_config; memset(&new_path_config, 0, sizeof(new_path_config)); calculate_path_impl(&calculate, &new_path_config, main_config); path_config = new_path_config; + if (use_tmp) { + _PyMainInterpreterConfig_Clear(&tmp_config); + } + calculate_free(&calculate); + return; + +fatal_error: + if (use_tmp) { + _PyMainInterpreterConfig_Clear(&tmp_config); + } calculate_free(&calculate); + _Py_FatalInitError(err); } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index b079990c40f2bd..e36b6c1b057180 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -796,7 +796,7 @@ _Py_InitializeCore(const _PyCoreConfig *config) */ _PyInitError -_Py_ReadMainInterpreterConfig(_PyMainInterpreterConfig *config) +_PyMainInterpreterConfig_Read(_PyMainInterpreterConfig *config) { /* Signal handlers are installed by default */ if (config->install_signal_handlers < 0) { @@ -805,6 +805,17 @@ _Py_ReadMainInterpreterConfig(_PyMainInterpreterConfig *config) return _Py_INIT_OK(); } + +void +_PyMainInterpreterConfig_Clear(_PyMainInterpreterConfig *config) +{ + PyMem_RawFree(config->module_search_path_env); + config->module_search_path_env = NULL; + PyMem_RawFree(config->home); + config->home = NULL; +} + + /* Update interpreter state based on supplied configuration settings * * After calling this function, most of the restrictions on the interpreter @@ -943,7 +954,7 @@ _Py_InitializeEx_Private(int install_sigs, int install_importlib) } /* TODO: Print any exceptions raised by these operations */ - err = _Py_ReadMainInterpreterConfig(&config); + err = _PyMainInterpreterConfig_Read(&config); if (_Py_INIT_FAILED(err)) { return err; } @@ -1477,8 +1488,8 @@ Py_SetPythonHome(wchar_t *home) } -_PyInitError -_Py_GetPythonHomeWithConfig(const _PyMainInterpreterConfig *config, wchar_t **homep) +wchar_t* +Py_GetPythonHome(void) { /* Use a static buffer to avoid heap memory allocation failure. Py_GetPythonHome() doesn't allow to report error, and the caller @@ -1486,40 +1497,22 @@ _Py_GetPythonHomeWithConfig(const _PyMainInterpreterConfig *config, wchar_t **ho static wchar_t buffer[MAXPATHLEN+1]; if (default_home) { - *homep = default_home; - return _Py_INIT_OK(); - } - - if (config) { - *homep = config->pythonhome; - return _Py_INIT_OK(); + return default_home; } char *home = Py_GETENV("PYTHONHOME"); if (!home) { - *homep = NULL; - return _Py_INIT_OK(); + return NULL; } size_t size = Py_ARRAY_LENGTH(buffer); size_t r = mbstowcs(buffer, home, size); if (r == (size_t)-1 || r >= size) { /* conversion failed or the static buffer is too small */ - *homep = NULL; - return _Py_INIT_ERR("failed to decode PYTHONHOME environment variable"); + return NULL; } - *homep = buffer; - return _Py_INIT_OK(); -} - -wchar_t * -Py_GetPythonHome(void) -{ - wchar_t *home; - /* Ignore error */ - (void)_Py_GetPythonHomeWithConfig(NULL, &home); - return home; + return buffer; } /* Add the __main__ module */ From f04ebe2a4d68b194deeb438e9185efdafc10a832 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 25 Nov 2017 00:01:23 +0100 Subject: [PATCH 36/75] bpo-32030: Add _PyMainInterpreterConfig.program_name (#4548) * Py_Main() now calls Py_SetProgramName() earlier to be able to get the program name in _PyMainInterpreterConfig_ReadEnv(). * Rename prog to program_name * Rename progpath to program_name --- Include/pystate.h | 2 ++ Modules/getpath.c | 60 +++++++++++++++++++++++-------------------- Modules/main.c | 21 +++++++++++---- PC/getpathp.c | 61 ++++++++++++++++++++------------------------ Python/pylifecycle.c | 22 +++++++++++++--- 5 files changed, 96 insertions(+), 70 deletions(-) diff --git a/Include/pystate.h b/Include/pystate.h index 533851fc0283af..60d001c4926c20 100644 --- a/Include/pystate.h +++ b/Include/pystate.h @@ -64,6 +64,8 @@ typedef struct { wchar_t *module_search_path_env; /* PYTHONHOME environment variable, see also Py_SetPythonHome(). */ wchar_t *home; + /* Program name, see also Py_GetProgramName() */ + wchar_t *program_name; } _PyMainInterpreterConfig; #define _PyMainInterpreterConfig_INIT \ diff --git a/Modules/getpath.c b/Modules/getpath.c index db28eaf66df2bc..9078aa81b3278d 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -112,7 +112,7 @@ extern "C" { typedef struct { wchar_t prefix[MAXPATHLEN+1]; wchar_t exec_prefix[MAXPATHLEN+1]; - wchar_t progpath[MAXPATHLEN+1]; + wchar_t program_name[MAXPATHLEN+1]; wchar_t *module_search_path; } PyPathConfig; @@ -121,7 +121,7 @@ typedef struct { wchar_t *home; /* PYTHONHOME environment variable */ wchar_t *module_search_path_env; /* PYTHONPATH environment variable */ - wchar_t *prog; /* Program name */ + wchar_t *program_name; /* Program name */ wchar_t *pythonpath; /* PYTHONPATH define */ wchar_t *prefix; /* PREFIX define */ wchar_t *exec_prefix; /* EXEC_PREFIX define */ @@ -602,8 +602,8 @@ calculate_progpath(PyCalculatePath *calculate, PyPathConfig *config) * other way to find a directory to start the search from. If * $PATH isn't exported, you lose. */ - if (wcschr(calculate->prog, SEP)) { - wcsncpy(config->progpath, calculate->prog, MAXPATHLEN); + if (wcschr(calculate->program_name, SEP)) { + wcsncpy(config->program_name, calculate->program_name, MAXPATHLEN); } #ifdef __APPLE__ /* On Mac OS X, if a script uses an interpreter of the form @@ -616,11 +616,13 @@ calculate_progpath(PyCalculatePath *calculate, PyPathConfig *config) * will fail if a relative path was used. but in that case, * absolutize() should help us out below */ - else if(0 == _NSGetExecutablePath(execpath, &nsexeclength) && execpath[0] == SEP) { - size_t r = mbstowcs(config->progpath, execpath, MAXPATHLEN+1); + else if(0 == _NSGetExecutablePath(execpath, &nsexeclength) && + execpath[0] == SEP) + { + size_t r = mbstowcs(config->program_name, execpath, MAXPATHLEN+1); if (r == (size_t)-1 || r > MAXPATHLEN) { /* Could not convert execpath, or it's too long. */ - config->progpath[0] = '\0'; + config->program_name[0] = '\0'; } } #endif /* __APPLE__ */ @@ -634,30 +636,30 @@ calculate_progpath(PyCalculatePath *calculate, PyPathConfig *config) if (len > MAXPATHLEN) { len = MAXPATHLEN; } - wcsncpy(config->progpath, path, len); - *(config->progpath + len) = '\0'; + wcsncpy(config->program_name, path, len); + *(config->program_name + len) = '\0'; } else { - wcsncpy(config->progpath, path, MAXPATHLEN); + wcsncpy(config->program_name, path, MAXPATHLEN); } - joinpath(config->progpath, calculate->prog); - if (isxfile(config->progpath)) { + joinpath(config->program_name, calculate->program_name); + if (isxfile(config->program_name)) { break; } if (!delim) { - config->progpath[0] = L'\0'; + config->program_name[0] = L'\0'; break; } path = delim + 1; } } else { - config->progpath[0] = '\0'; + config->program_name[0] = '\0'; } - if (config->progpath[0] != SEP && config->progpath[0] != '\0') { - absolutize(config->progpath); + if (config->program_name[0] != SEP && config->program_name[0] != '\0') { + absolutize(config->program_name); } } @@ -665,7 +667,7 @@ calculate_progpath(PyCalculatePath *calculate, PyPathConfig *config) static void calculate_argv0_path(PyCalculatePath *calculate, PyPathConfig *config) { - wcsncpy(calculate->argv0_path, config->progpath, MAXPATHLEN); + wcsncpy(calculate->argv0_path, config->program_name, MAXPATHLEN); calculate->argv0_path[MAXPATHLEN] = '\0'; #ifdef WITH_NEXT_FRAMEWORK @@ -700,10 +702,10 @@ calculate_argv0_path(PyCalculatePath *calculate, PyPathConfig *config) if (!ismodule(calculate->argv0_path)) { /* We are in the build directory so use the name of the executable - we know that the absolute path is passed */ - wcsncpy(calculate->argv0_path, config->progpath, MAXPATHLEN); + wcsncpy(calculate->argv0_path, config->program_name, MAXPATHLEN); } else { - /* Use the location of the library as the progpath */ + /* Use the location of the library as the program_name */ wcsncpy(calculate->argv0_path, wbuf, MAXPATHLEN); } PyMem_RawFree(wbuf); @@ -712,7 +714,7 @@ calculate_argv0_path(PyCalculatePath *calculate, PyPathConfig *config) #if HAVE_READLINK wchar_t tmpbuffer[MAXPATHLEN+1]; - int linklen = _Py_wreadlink(config->progpath, tmpbuffer, MAXPATHLEN); + int linklen = _Py_wreadlink(config->program_name, tmpbuffer, MAXPATHLEN); while (linklen != -1) { if (tmpbuffer[0] == SEP) { /* tmpbuffer should never be longer than MAXPATHLEN, @@ -720,7 +722,7 @@ calculate_argv0_path(PyCalculatePath *calculate, PyPathConfig *config) wcsncpy(calculate->argv0_path, tmpbuffer, MAXPATHLEN); } else { - /* Interpret relative to progpath */ + /* Interpret relative to program_name */ reduce(calculate->argv0_path); joinpath(calculate->argv0_path, tmpbuffer); } @@ -897,6 +899,7 @@ calculate_init(PyCalculatePath *calculate, { calculate->home = main_config->home; calculate->module_search_path_env = main_config->module_search_path_env; + calculate->program_name = main_config->program_name; size_t len; char *path = getenv("PATH"); @@ -907,8 +910,6 @@ calculate_init(PyCalculatePath *calculate, } } - calculate->prog = Py_GetProgramName(); - calculate->pythonpath = Py_DecodeLocale(PYTHONPATH, &len); if (!calculate->pythonpath) { return DECODE_LOCALE_ERR("PYTHONPATH define", len); @@ -950,7 +951,9 @@ calculate_path_impl(PyCalculatePath *calculate, PyPathConfig *config) calculate_zip_path(calculate, config); calculate_exec_prefix(calculate, config); - if ((!calculate->prefix_found || !calculate->exec_prefix_found) && !Py_FrozenFlag) { + if ((!calculate->prefix_found || !calculate->exec_prefix_found) && + !Py_FrozenFlag) + { fprintf(stderr, "Consider setting $PYTHONHOME to [:]\n"); } @@ -1018,10 +1021,11 @@ Py_SetPath(const wchar_t *path) return; } - wchar_t *prog = Py_GetProgramName(); - wcsncpy(path_config.progpath, prog, MAXPATHLEN); + wchar_t *program_name = Py_GetProgramName(); + wcsncpy(path_config.program_name, program_name, MAXPATHLEN); path_config.exec_prefix[0] = path_config.prefix[0] = L'\0'; - path_config.module_search_path = PyMem_RawMalloc((wcslen(path) + 1) * sizeof(wchar_t)); + size_t size = (wcslen(path) + 1) * sizeof(wchar_t); + path_config.module_search_path = PyMem_RawMalloc(size); if (path_config.module_search_path != NULL) { wcscpy(path_config.module_search_path, path); } @@ -1074,7 +1078,7 @@ Py_GetProgramFullPath(void) if (!path_config.module_search_path) { calculate_path(NULL); } - return path_config.progpath; + return path_config.program_name; } #ifdef __cplusplus diff --git a/Modules/main.c b/Modules/main.c index 5b0c04938411e6..dca165dad2c41a 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -1452,6 +1452,13 @@ _PyMainInterpreterConfig_ReadEnv(_PyMainInterpreterConfig *config) return err; } + /* FIXME: _PyMainInterpreterConfig_Read() has the same code. Remove it + here? See also pymain_get_program_name() and pymain_parse_envvars(). */ + config->program_name = _PyMem_RawWcsdup(Py_GetProgramName()); + if (config->program_name == NULL) { + return _Py_INIT_NO_MEMORY(); + } + return _Py_INIT_OK(); } @@ -1480,6 +1487,15 @@ pymain_parse_envvars(_PyMain *pymain) } core_config->allocator = Py_GETENV("PYTHONMALLOC"); + /* FIXME: move pymain_get_program_name() code into + _PyMainInterpreterConfig_ReadEnv(). + Problem: _PyMainInterpreterConfig_ReadEnv() doesn't have access + to argv[0]. */ + Py_SetProgramName(pymain->program_name); + /* Don't free program_name here: the argument to Py_SetProgramName + must remain valid until Py_FinalizeEx is called. The string is freed + by pymain_free(). */ + _PyInitError err = _PyMainInterpreterConfig_ReadEnv(&pymain->config); if (_Py_INIT_FAILED(pymain->err)) { pymain->err = err; @@ -1569,11 +1585,6 @@ pymain_init_python(_PyMain *pymain) return -1; } - Py_SetProgramName(pymain->program_name); - /* Don't free program_name here: the argument to Py_SetProgramName - must remain valid until Py_FinalizeEx is called. The string is freed - by pymain_free(). */ - if (pymain_add_xoptions(pymain)) { return -1; } diff --git a/PC/getpathp.c b/PC/getpathp.c index 5adf16d6f27384..38e433b6dcd05a 100644 --- a/PC/getpathp.c +++ b/PC/getpathp.c @@ -118,7 +118,7 @@ typedef struct { wchar_t prefix[MAXPATHLEN+1]; - wchar_t progpath[MAXPATHLEN+1]; + wchar_t program_name[MAXPATHLEN+1]; wchar_t dllpath[MAXPATHLEN+1]; wchar_t *module_search_path; } PyPathConfig; @@ -132,7 +132,7 @@ typedef struct { wchar_t *machine_path; /* from HKEY_LOCAL_MACHINE */ wchar_t *user_path; /* from HKEY_CURRENT_USER */ - wchar_t *prog; /* Program name */ + wchar_t *program_name; /* Program name */ wchar_t argv0_path[MAXPATHLEN+1]; wchar_t zip_path[MAXPATHLEN+1]; } PyCalculatePath; @@ -484,22 +484,22 @@ getpythonregpath(HKEY keyBase, int skipcore) static void -get_progpath(PyCalculatePath *calculate, wchar_t *progpath, wchar_t *dllpath) +get_progpath(PyCalculatePath *calculate, PyPathConfig *config) { wchar_t *path = calculate->path_env; #ifdef Py_ENABLE_SHARED extern HANDLE PyWin_DLLhModule; - /* static init of progpath ensures final char remains \0 */ + /* static init of program_name ensures final char remains \0 */ if (PyWin_DLLhModule) { - if (!GetModuleFileNameW(PyWin_DLLhModule, dllpath, MAXPATHLEN)) { - dllpath[0] = 0; + if (!GetModuleFileNameW(PyWin_DLLhModule, config->dllpath, MAXPATHLEN)) { + config->dllpath[0] = 0; } } #else - dllpath[0] = 0; + config->dllpath[0] = 0; #endif - if (GetModuleFileNameW(NULL, progpath, MAXPATHLEN)) { + if (GetModuleFileNameW(NULL, config->program_name, MAXPATHLEN)) { return; } @@ -509,12 +509,12 @@ get_progpath(PyCalculatePath *calculate, wchar_t *progpath, wchar_t *dllpath) * $PATH isn't exported, you lose. */ #ifdef ALTSEP - if (wcschr(calculate->prog, SEP) || wcschr(calculate->prog, ALTSEP)) + if (wcschr(calculate->program_name, SEP) || wcschr(calculate->program_name, ALTSEP)) #else - if (wcschr(calculate->prog, SEP)) + if (wcschr(calculate->program_name, SEP)) #endif { - wcsncpy(progpath, calculate->prog, MAXPATHLEN); + wcsncpy(config->program_name, calculate->program_name, MAXPATHLEN); } else if (path) { while (1) { @@ -524,28 +524,28 @@ get_progpath(PyCalculatePath *calculate, wchar_t *progpath, wchar_t *dllpath) size_t len = delim - path; /* ensure we can't overwrite buffer */ len = min(MAXPATHLEN,len); - wcsncpy(progpath, path, len); - *(progpath + len) = '\0'; + wcsncpy(config->program_name, path, len); + *(config->program_name + len) = '\0'; } else { - wcsncpy(progpath, path, MAXPATHLEN); + wcsncpy(config->program_name, path, MAXPATHLEN); } /* join() is safe for MAXPATHLEN+1 size buffer */ - join(progpath, calculate->prog); - if (exists(progpath)) { + join(config->program_name, calculate->program_name); + if (exists(config->program_name)) { break; } if (!delim) { - progpath[0] = '\0'; + config->program_name[0] = '\0'; break; } path = delim + 1; } } else { - progpath[0] = '\0'; + config->program_name[0] = '\0'; } } @@ -695,14 +695,9 @@ calculate_init(PyCalculatePath *calculate, { calculate->home = main_config->home; calculate->module_search_path_env = main_config->module_search_path_env; + calculate->program_name = main_config->program_name; calculate->path_env = _wgetenv(L"PATH"); - - wchar_t *prog = Py_GetProgramName(); - if (prog == NULL || *prog == '\0') { - prog = L"python"; - } - calculate->prog = prog; } @@ -714,8 +709,8 @@ get_pth_filename(wchar_t *spbuffer, PyPathConfig *config) return 1; } } - if (config->progpath[0]) { - if (!change_ext(spbuffer, config->progpath, L"._pth") && exists(spbuffer)) { + if (config->program_name[0]) { + if (!change_ext(spbuffer, config->program_name, L"._pth") && exists(spbuffer)) { return 1; } } @@ -784,9 +779,9 @@ static void calculate_path_impl(PyCalculatePath *calculate, PyPathConfig *config, const _PyMainInterpreterConfig *main_config) { - get_progpath(calculate, config->progpath, config->dllpath); - /* progpath guaranteed \0 terminated in MAXPATH+1 bytes. */ - wcscpy_s(calculate->argv0_path, MAXPATHLEN+1, config->progpath); + get_progpath(calculate, config); + /* program_name guaranteed \0 terminated in MAXPATH+1 bytes. */ + wcscpy_s(calculate->argv0_path, MAXPATHLEN+1, config->program_name); reduce(calculate->argv0_path); /* Search for a sys.path file */ @@ -798,7 +793,7 @@ calculate_path_impl(PyCalculatePath *calculate, PyPathConfig *config, /* Calculate zip archive path from DLL or exe path */ change_ext(calculate->zip_path, - config->dllpath[0] ? config->dllpath : config->progpath, + config->dllpath[0] ? config->dllpath : config->program_name, L".zip"); if (calculate->home == NULL || *calculate->home == '\0') { @@ -1057,8 +1052,8 @@ Py_SetPath(const wchar_t *path) return; } - wchar_t *prog = Py_GetProgramName(); - wcsncpy(path_config.progpath, prog, MAXPATHLEN); + wchar_t *program_name = Py_GetProgramName(); + wcsncpy(path_config.program_name, program_name, MAXPATHLEN); path_config.prefix[0] = L'\0'; path_config.module_search_path = PyMem_RawMalloc((wcslen(path) + 1) * sizeof(wchar_t)); if (path_config.module_search_path != NULL) { @@ -1110,7 +1105,7 @@ Py_GetProgramFullPath(void) if (!path_config.module_search_path) { calculate_path(NULL); } - return path_config.progpath; + return path_config.program_name; } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index e36b6c1b057180..714be3768f6ba3 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -802,6 +802,14 @@ _PyMainInterpreterConfig_Read(_PyMainInterpreterConfig *config) if (config->install_signal_handlers < 0) { config->install_signal_handlers = 1; } + + if (config->program_name == NULL) { + config->program_name = _PyMem_RawWcsdup(Py_GetProgramName()); + if (config->program_name == NULL) { + return _Py_INIT_NO_MEMORY(); + } + } + return _Py_INIT_OK(); } @@ -809,10 +817,16 @@ _PyMainInterpreterConfig_Read(_PyMainInterpreterConfig *config) void _PyMainInterpreterConfig_Clear(_PyMainInterpreterConfig *config) { - PyMem_RawFree(config->module_search_path_env); - config->module_search_path_env = NULL; - PyMem_RawFree(config->home); - config->home = NULL; +#define CLEAR(ATTR) \ + do { \ + PyMem_RawFree(ATTR); \ + ATTR = NULL; \ + } while (0) + + CLEAR(config->module_search_path_env); + CLEAR(config->home); + CLEAR(config->program_name); +#undef CLEAR } From f8802d80b32dbc64f9e0e72270695d24ac50e246 Mon Sep 17 00:00:00 2001 From: Ashley Camba Date: Sat, 25 Nov 2017 00:39:39 +0100 Subject: [PATCH 37/75] Asyncion-Dev docs: Fix the reference to sys.excepthook (GH-4414) --- Doc/library/asyncio-dev.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/asyncio-dev.rst b/Doc/library/asyncio-dev.rst index b2ad87b5d02057..b9735de2078d18 100644 --- a/Doc/library/asyncio-dev.rst +++ b/Doc/library/asyncio-dev.rst @@ -216,9 +216,9 @@ The fix is to call the :func:`ensure_future` function or the Detect exceptions never consumed -------------------------------- -Python usually calls :func:`sys.displayhook` on unhandled exceptions. If +Python usually calls :func:`sys.excepthook` on unhandled exceptions. If :meth:`Future.set_exception` is called, but the exception is never consumed, -:func:`sys.displayhook` is not called. Instead, :ref:`a log is emitted +:func:`sys.excepthook` is not called. Instead, :ref:`a log is emitted ` when the future is deleted by the garbage collector, with the traceback where the exception was raised. From 706cb3162e15271ecfeba15909ed48a3a437009f Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 25 Nov 2017 02:42:18 +0100 Subject: [PATCH 38/75] bpo-32128: Skip test_nntplib.test_article_head_body() (#4552) The NNTP server currently has troubles with SSL, whereas we don't have the control on this server. This test blocks all CIs, so disable it until a fix can be found. --- Lib/test/test_nntplib.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_nntplib.py b/Lib/test/test_nntplib.py index bb780bfa004b57..8c1032b986bf61 100644 --- a/Lib/test/test_nntplib.py +++ b/Lib/test/test_nntplib.py @@ -165,6 +165,7 @@ def check_article_resp(self, resp, article, art_num=None): # XXX this could exceptionally happen... self.assertNotIn(article.lines[-1], (b".", b".\n", b".\r\n")) + @unittest.skipIf(True, "FIXME: see bpo-32128") def test_article_head_body(self): resp, count, first, last, name = self.server.group(self.GROUP_NAME) # Try to find an available article From 9316ee4da2dcc217351418fc4dbe9205995689e0 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 25 Nov 2017 03:17:57 +0100 Subject: [PATCH 39/75] bpo-32030: Add _PyPathConfig_Init() (#4551) * Add _PyPathConfig_Init() and _PyPathConfig_Fini() * Remove _Py_GetPathWithConfig() * _PyPathConfig_Init() returns _PyInitError to allow to handle errors properly * Add pathconfig_clear() * Windows calculate_path_impl(): replace Py_FatalError() with _PyInitError * Py_FinalizeEx() now calls _PyPathConfig_Fini() to release memory * Fix _Py_InitializeMainInterpreter() regression: don't initialize path config if _disable_importlib is false * PyPathConfig now uses dynamically allocated memory --- Include/pylifecycle.h | 5 +- Modules/getpath.c | 428 ++++++++++++++++++++++++------------------ Modules/main.c | 4 +- PC/getpathp.c | 318 ++++++++++++++++++++----------- Python/pylifecycle.c | 28 ++- 5 files changed, 476 insertions(+), 307 deletions(-) diff --git a/Include/pylifecycle.h b/Include/pylifecycle.h index ff915323386594..d32c98b6985688 100644 --- a/Include/pylifecycle.h +++ b/Include/pylifecycle.h @@ -103,8 +103,9 @@ PyAPI_FUNC(wchar_t *) Py_GetPrefix(void); PyAPI_FUNC(wchar_t *) Py_GetExecPrefix(void); PyAPI_FUNC(wchar_t *) Py_GetPath(void); #ifdef Py_BUILD_CORE -PyAPI_FUNC(wchar_t *) _Py_GetPathWithConfig( - const _PyMainInterpreterConfig *config); +PyAPI_FUNC(_PyInitError) _PyPathConfig_Init( + const _PyMainInterpreterConfig *main_config); +PyAPI_FUNC(void) _PyPathConfig_Fini(void); #endif PyAPI_FUNC(void) Py_SetPath(const wchar_t *); #ifdef MS_WINDOWS diff --git a/Modules/getpath.c b/Modules/getpath.c index 9078aa81b3278d..02d626cea73b38 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -109,10 +109,16 @@ extern "C" { #define LANDMARK L"os.py" #endif +#define DECODE_LOCALE_ERR(NAME, LEN) \ + ((LEN) == (size_t)-2) \ + ? _Py_INIT_USER_ERR("cannot decode " #NAME) \ + : _Py_INIT_NO_MEMORY() + + typedef struct { - wchar_t prefix[MAXPATHLEN+1]; - wchar_t exec_prefix[MAXPATHLEN+1]; - wchar_t program_name[MAXPATHLEN+1]; + wchar_t *prefix; + wchar_t *exec_prefix; + wchar_t *program_name; wchar_t *module_search_path; } PyPathConfig; @@ -361,63 +367,63 @@ find_env_config_value(FILE * env_file, const wchar_t * key, wchar_t * value) bytes long. */ static int -search_for_prefix(PyCalculatePath *calculate, PyPathConfig *config) +search_for_prefix(PyCalculatePath *calculate, wchar_t *prefix) { size_t n; wchar_t *vpath; /* If PYTHONHOME is set, we believe it unconditionally */ if (calculate->home) { - wcsncpy(config->prefix, calculate->home, MAXPATHLEN); - config->prefix[MAXPATHLEN] = L'\0'; - wchar_t *delim = wcschr(config->prefix, DELIM); + wcsncpy(prefix, calculate->home, MAXPATHLEN); + prefix[MAXPATHLEN] = L'\0'; + wchar_t *delim = wcschr(prefix, DELIM); if (delim) { *delim = L'\0'; } - joinpath(config->prefix, calculate->lib_python); - joinpath(config->prefix, LANDMARK); + joinpath(prefix, calculate->lib_python); + joinpath(prefix, LANDMARK); return 1; } /* Check to see if argv[0] is in the build directory */ - wcsncpy(config->prefix, calculate->argv0_path, MAXPATHLEN); - config->prefix[MAXPATHLEN] = L'\0'; - joinpath(config->prefix, L"Modules/Setup"); - if (isfile(config->prefix)) { + wcsncpy(prefix, calculate->argv0_path, MAXPATHLEN); + prefix[MAXPATHLEN] = L'\0'; + joinpath(prefix, L"Modules/Setup"); + if (isfile(prefix)) { /* Check VPATH to see if argv0_path is in the build directory. */ vpath = Py_DecodeLocale(VPATH, NULL); if (vpath != NULL) { - wcsncpy(config->prefix, calculate->argv0_path, MAXPATHLEN); - config->prefix[MAXPATHLEN] = L'\0'; - joinpath(config->prefix, vpath); + wcsncpy(prefix, calculate->argv0_path, MAXPATHLEN); + prefix[MAXPATHLEN] = L'\0'; + joinpath(prefix, vpath); PyMem_RawFree(vpath); - joinpath(config->prefix, L"Lib"); - joinpath(config->prefix, LANDMARK); - if (ismodule(config->prefix)) { + joinpath(prefix, L"Lib"); + joinpath(prefix, LANDMARK); + if (ismodule(prefix)) { return -1; } } } /* Search from argv0_path, until root is found */ - copy_absolute(config->prefix, calculate->argv0_path, MAXPATHLEN+1); + copy_absolute(prefix, calculate->argv0_path, MAXPATHLEN+1); do { - n = wcslen(config->prefix); - joinpath(config->prefix, calculate->lib_python); - joinpath(config->prefix, LANDMARK); - if (ismodule(config->prefix)) { + n = wcslen(prefix); + joinpath(prefix, calculate->lib_python); + joinpath(prefix, LANDMARK); + if (ismodule(prefix)) { return 1; } - config->prefix[n] = L'\0'; - reduce(config->prefix); - } while (config->prefix[0]); + prefix[n] = L'\0'; + reduce(prefix); + } while (prefix[0]); /* Look at configure's PREFIX */ - wcsncpy(config->prefix, calculate->prefix, MAXPATHLEN); - config->prefix[MAXPATHLEN] = L'\0'; - joinpath(config->prefix, calculate->lib_python); - joinpath(config->prefix, LANDMARK); - if (ismodule(config->prefix)) { + wcsncpy(prefix, calculate->prefix, MAXPATHLEN); + prefix[MAXPATHLEN] = L'\0'; + joinpath(prefix, calculate->lib_python); + joinpath(prefix, LANDMARK); + if (ismodule(prefix)) { return 1; } @@ -427,25 +433,25 @@ search_for_prefix(PyCalculatePath *calculate, PyPathConfig *config) static void -calculate_prefix(PyCalculatePath *calculate, PyPathConfig *config) +calculate_prefix(PyCalculatePath *calculate, wchar_t *prefix) { - calculate->prefix_found = search_for_prefix(calculate, config); + calculate->prefix_found = search_for_prefix(calculate, prefix); if (!calculate->prefix_found) { if (!Py_FrozenFlag) { fprintf(stderr, "Could not find platform independent libraries \n"); } - wcsncpy(config->prefix, calculate->prefix, MAXPATHLEN); - joinpath(config->prefix, calculate->lib_python); + wcsncpy(prefix, calculate->prefix, MAXPATHLEN); + joinpath(prefix, calculate->lib_python); } else { - reduce(config->prefix); + reduce(prefix); } } static void -calculate_reduce_prefix(PyCalculatePath *calculate, PyPathConfig *config) +calculate_reduce_prefix(PyCalculatePath *calculate, wchar_t *prefix) { /* Reduce prefix and exec_prefix to their essence, * e.g. /usr/local/lib/python1.5 is reduced to /usr/local. @@ -453,16 +459,16 @@ calculate_reduce_prefix(PyCalculatePath *calculate, PyPathConfig *config) * return the compiled-in defaults instead. */ if (calculate->prefix_found > 0) { - reduce(config->prefix); - reduce(config->prefix); + reduce(prefix); + reduce(prefix); /* The prefix is the root directory, but reduce() chopped * off the "/". */ - if (!config->prefix[0]) { - wcscpy(config->prefix, separator); + if (!prefix[0]) { + wcscpy(prefix, separator); } } else { - wcsncpy(config->prefix, calculate->prefix, MAXPATHLEN); + wcsncpy(prefix, calculate->prefix, MAXPATHLEN); } } @@ -471,7 +477,7 @@ calculate_reduce_prefix(PyCalculatePath *calculate, PyPathConfig *config) MAXPATHLEN bytes long. */ static int -search_for_exec_prefix(PyCalculatePath *calculate, PyPathConfig *config) +search_for_exec_prefix(PyCalculatePath *calculate, wchar_t *exec_prefix) { size_t n; @@ -479,25 +485,25 @@ search_for_exec_prefix(PyCalculatePath *calculate, PyPathConfig *config) if (calculate->home) { wchar_t *delim = wcschr(calculate->home, DELIM); if (delim) { - wcsncpy(config->exec_prefix, delim+1, MAXPATHLEN); + wcsncpy(exec_prefix, delim+1, MAXPATHLEN); } else { - wcsncpy(config->exec_prefix, calculate->home, MAXPATHLEN); + wcsncpy(exec_prefix, calculate->home, MAXPATHLEN); } - config->exec_prefix[MAXPATHLEN] = L'\0'; - joinpath(config->exec_prefix, calculate->lib_python); - joinpath(config->exec_prefix, L"lib-dynload"); + exec_prefix[MAXPATHLEN] = L'\0'; + joinpath(exec_prefix, calculate->lib_python); + joinpath(exec_prefix, L"lib-dynload"); return 1; } /* Check to see if argv[0] is in the build directory. "pybuilddir.txt" is written by setup.py and contains the relative path to the location of shared library modules. */ - wcsncpy(config->exec_prefix, calculate->argv0_path, MAXPATHLEN); - config->exec_prefix[MAXPATHLEN] = L'\0'; - joinpath(config->exec_prefix, L"pybuilddir.txt"); - if (isfile(config->exec_prefix)) { - FILE *f = _Py_wfopen(config->exec_prefix, L"rb"); + wcsncpy(exec_prefix, calculate->argv0_path, MAXPATHLEN); + exec_prefix[MAXPATHLEN] = L'\0'; + joinpath(exec_prefix, L"pybuilddir.txt"); + if (isfile(exec_prefix)) { + FILE *f = _Py_wfopen(exec_prefix, L"rb"); if (f == NULL) { errno = 0; } @@ -516,9 +522,9 @@ search_for_exec_prefix(PyCalculatePath *calculate, PyPathConfig *config) Py_DECREF(decoded); if (k >= 0) { rel_builddir_path[k] = L'\0'; - wcsncpy(config->exec_prefix, calculate->argv0_path, MAXPATHLEN); - config->exec_prefix[MAXPATHLEN] = L'\0'; - joinpath(config->exec_prefix, rel_builddir_path); + wcsncpy(exec_prefix, calculate->argv0_path, MAXPATHLEN); + exec_prefix[MAXPATHLEN] = L'\0'; + joinpath(exec_prefix, rel_builddir_path); return -1; } } @@ -526,24 +532,24 @@ search_for_exec_prefix(PyCalculatePath *calculate, PyPathConfig *config) } /* Search from argv0_path, until root is found */ - copy_absolute(config->exec_prefix, calculate->argv0_path, MAXPATHLEN+1); + copy_absolute(exec_prefix, calculate->argv0_path, MAXPATHLEN+1); do { - n = wcslen(config->exec_prefix); - joinpath(config->exec_prefix, calculate->lib_python); - joinpath(config->exec_prefix, L"lib-dynload"); - if (isdir(config->exec_prefix)) { + n = wcslen(exec_prefix); + joinpath(exec_prefix, calculate->lib_python); + joinpath(exec_prefix, L"lib-dynload"); + if (isdir(exec_prefix)) { return 1; } - config->exec_prefix[n] = L'\0'; - reduce(config->exec_prefix); - } while (config->exec_prefix[0]); + exec_prefix[n] = L'\0'; + reduce(exec_prefix); + } while (exec_prefix[0]); /* Look at configure's EXEC_PREFIX */ - wcsncpy(config->exec_prefix, calculate->exec_prefix, MAXPATHLEN); - config->exec_prefix[MAXPATHLEN] = L'\0'; - joinpath(config->exec_prefix, calculate->lib_python); - joinpath(config->exec_prefix, L"lib-dynload"); - if (isdir(config->exec_prefix)) { + wcsncpy(exec_prefix, calculate->exec_prefix, MAXPATHLEN); + exec_prefix[MAXPATHLEN] = L'\0'; + joinpath(exec_prefix, calculate->lib_python); + joinpath(exec_prefix, L"lib-dynload"); + if (isdir(exec_prefix)) { return 1; } @@ -553,41 +559,44 @@ search_for_exec_prefix(PyCalculatePath *calculate, PyPathConfig *config) static void -calculate_exec_prefix(PyCalculatePath *calculate, PyPathConfig *config) +calculate_exec_prefix(PyCalculatePath *calculate, wchar_t *exec_prefix) { - calculate->exec_prefix_found = search_for_exec_prefix(calculate, config); + calculate->exec_prefix_found = search_for_exec_prefix(calculate, exec_prefix); if (!calculate->exec_prefix_found) { if (!Py_FrozenFlag) { fprintf(stderr, "Could not find platform dependent libraries \n"); } - wcsncpy(config->exec_prefix, calculate->exec_prefix, MAXPATHLEN); - joinpath(config->exec_prefix, L"lib/lib-dynload"); + wcsncpy(exec_prefix, calculate->exec_prefix, MAXPATHLEN); + joinpath(exec_prefix, L"lib/lib-dynload"); } /* If we found EXEC_PREFIX do *not* reduce it! (Yet.) */ } static void -calculate_reduce_exec_prefix(PyCalculatePath *calculate, PyPathConfig *config) +calculate_reduce_exec_prefix(PyCalculatePath *calculate, wchar_t *exec_prefix) { if (calculate->exec_prefix_found > 0) { - reduce(config->exec_prefix); - reduce(config->exec_prefix); - reduce(config->exec_prefix); - if (!config->exec_prefix[0]) { - wcscpy(config->exec_prefix, separator); + reduce(exec_prefix); + reduce(exec_prefix); + reduce(exec_prefix); + if (!exec_prefix[0]) { + wcscpy(exec_prefix, separator); } } else { - wcsncpy(config->exec_prefix, calculate->exec_prefix, MAXPATHLEN); + wcsncpy(exec_prefix, calculate->exec_prefix, MAXPATHLEN); } } -static void -calculate_progpath(PyCalculatePath *calculate, PyPathConfig *config) +static _PyInitError +calculate_program_name(PyCalculatePath *calculate, PyPathConfig *config) { + wchar_t program_name[MAXPATHLEN+1]; + memset(program_name, 0, sizeof(program_name)); + #ifdef __APPLE__ #if MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_4 uint32_t nsexeclength = MAXPATHLEN; @@ -603,7 +612,7 @@ calculate_progpath(PyCalculatePath *calculate, PyPathConfig *config) * $PATH isn't exported, you lose. */ if (wcschr(calculate->program_name, SEP)) { - wcsncpy(config->program_name, calculate->program_name, MAXPATHLEN); + wcsncpy(program_name, calculate->program_name, MAXPATHLEN); } #ifdef __APPLE__ /* On Mac OS X, if a script uses an interpreter of the form @@ -619,10 +628,10 @@ calculate_progpath(PyCalculatePath *calculate, PyPathConfig *config) else if(0 == _NSGetExecutablePath(execpath, &nsexeclength) && execpath[0] == SEP) { - size_t r = mbstowcs(config->program_name, execpath, MAXPATHLEN+1); + size_t r = mbstowcs(program_name, execpath, MAXPATHLEN+1); if (r == (size_t)-1 || r > MAXPATHLEN) { /* Could not convert execpath, or it's too long. */ - config->program_name[0] = '\0'; + program_name[0] = '\0'; } } #endif /* __APPLE__ */ @@ -636,38 +645,44 @@ calculate_progpath(PyCalculatePath *calculate, PyPathConfig *config) if (len > MAXPATHLEN) { len = MAXPATHLEN; } - wcsncpy(config->program_name, path, len); - *(config->program_name + len) = '\0'; + wcsncpy(program_name, path, len); + program_name[len] = '\0'; } else { - wcsncpy(config->program_name, path, MAXPATHLEN); + wcsncpy(program_name, path, MAXPATHLEN); } - joinpath(config->program_name, calculate->program_name); - if (isxfile(config->program_name)) { + joinpath(program_name, calculate->program_name); + if (isxfile(program_name)) { break; } if (!delim) { - config->program_name[0] = L'\0'; + program_name[0] = L'\0'; break; } path = delim + 1; } } else { - config->program_name[0] = '\0'; + program_name[0] = '\0'; + } + if (program_name[0] != SEP && program_name[0] != '\0') { + absolutize(program_name); } - if (config->program_name[0] != SEP && config->program_name[0] != '\0') { - absolutize(config->program_name); + + config->program_name = _PyMem_RawWcsdup(program_name); + if (config->program_name == NULL) { + return _Py_INIT_NO_MEMORY(); } + return _Py_INIT_OK(); } -static void -calculate_argv0_path(PyCalculatePath *calculate, PyPathConfig *config) +static _PyInitError +calculate_argv0_path(PyCalculatePath *calculate, const wchar_t *program_name) { - wcsncpy(calculate->argv0_path, config->program_name, MAXPATHLEN); + wcsncpy(calculate->argv0_path, program_name, MAXPATHLEN); calculate->argv0_path[MAXPATHLEN] = '\0'; #ifdef WITH_NEXT_FRAMEWORK @@ -690,9 +705,10 @@ calculate_argv0_path(PyCalculatePath *calculate, PyPathConfig *config) ** be running the interpreter in the build directory, so we use the ** build-directory-specific logic to find Lib and such. */ - wchar_t* wbuf = Py_DecodeLocale(modPath, NULL); + size_t len; + wchar_t* wbuf = Py_DecodeLocale(modPath, &len); if (wbuf == NULL) { - Py_FatalError("Cannot decode framework location"); + return DECODE_LOCALE_ERR("framework location", len); } wcsncpy(calculate->argv0_path, wbuf, MAXPATHLEN); @@ -702,7 +718,7 @@ calculate_argv0_path(PyCalculatePath *calculate, PyPathConfig *config) if (!ismodule(calculate->argv0_path)) { /* We are in the build directory so use the name of the executable - we know that the absolute path is passed */ - wcsncpy(calculate->argv0_path, config->program_name, MAXPATHLEN); + wcsncpy(calculate->argv0_path, program_name, MAXPATHLEN); } else { /* Use the location of the library as the program_name */ @@ -714,7 +730,7 @@ calculate_argv0_path(PyCalculatePath *calculate, PyPathConfig *config) #if HAVE_READLINK wchar_t tmpbuffer[MAXPATHLEN+1]; - int linklen = _Py_wreadlink(config->program_name, tmpbuffer, MAXPATHLEN); + int linklen = _Py_wreadlink(program_name, tmpbuffer, MAXPATHLEN); while (linklen != -1) { if (tmpbuffer[0] == SEP) { /* tmpbuffer should never be longer than MAXPATHLEN, @@ -733,6 +749,7 @@ calculate_argv0_path(PyCalculatePath *calculate, PyPathConfig *config) reduce(calculate->argv0_path); /* At this point, argv0_path is guaranteed to be less than MAXPATHLEN bytes long. */ + return _Py_INIT_OK(); } @@ -777,9 +794,9 @@ calculate_read_pyenv(PyCalculatePath *calculate) static void -calculate_zip_path(PyCalculatePath *calculate, PyPathConfig *config) +calculate_zip_path(PyCalculatePath *calculate, const wchar_t *prefix) { - wcsncpy(calculate->zip_path, config->prefix, MAXPATHLEN); + wcsncpy(calculate->zip_path, prefix, MAXPATHLEN); calculate->zip_path[MAXPATHLEN] = L'\0'; if (calculate->prefix_found > 0) { @@ -799,8 +816,10 @@ calculate_zip_path(PyCalculatePath *calculate, PyPathConfig *config) } -static wchar_t * -calculate_module_search_path(PyCalculatePath *calculate, PyPathConfig *config) +static _PyInitError +calculate_module_search_path(PyCalculatePath *calculate, + const wchar_t *prefix, const wchar_t *exec_prefix, + PyPathConfig *config) { /* Calculate size of return buffer */ size_t bufsz = 0; @@ -809,7 +828,7 @@ calculate_module_search_path(PyCalculatePath *calculate, PyPathConfig *config) } wchar_t *defpath = calculate->pythonpath; - size_t prefixsz = wcslen(config->prefix) + 1; + size_t prefixsz = wcslen(prefix) + 1; while (1) { wchar_t *delim = wcschr(defpath, DELIM); @@ -829,13 +848,12 @@ calculate_module_search_path(PyCalculatePath *calculate, PyPathConfig *config) } bufsz += wcslen(calculate->zip_path) + 1; - bufsz += wcslen(config->exec_prefix) + 1; + bufsz += wcslen(exec_prefix) + 1; /* Allocate the buffer */ wchar_t *buf = PyMem_RawMalloc(bufsz * sizeof(wchar_t)); if (buf == NULL) { - Py_FatalError( - "Not enough memory for dynamic PYTHONPATH"); + return _Py_INIT_NO_MEMORY(); } buf[0] = '\0'; @@ -857,8 +875,8 @@ calculate_module_search_path(PyCalculatePath *calculate, PyPathConfig *config) wchar_t *delim = wcschr(defpath, DELIM); if (defpath[0] != SEP) { - wcscat(buf, config->prefix); - if (prefixsz >= 2 && config->prefix[prefixsz - 2] != SEP && + wcscat(buf, prefix); + if (prefixsz >= 2 && prefix[prefixsz - 2] != SEP && defpath[0] != (delim ? DELIM : L'\0')) { /* not empty */ @@ -870,7 +888,7 @@ calculate_module_search_path(PyCalculatePath *calculate, PyPathConfig *config) size_t len = delim - defpath + 1; size_t end = wcslen(buf) + len; wcsncat(buf, defpath, len); - *(buf + end) = '\0'; + buf[end] = '\0'; } else { wcscat(buf, defpath); @@ -881,18 +899,13 @@ calculate_module_search_path(PyCalculatePath *calculate, PyPathConfig *config) wcscat(buf, delimiter); /* Finally, on goes the directory for dynamic-load modules */ - wcscat(buf, config->exec_prefix); + wcscat(buf, exec_prefix); - return buf; + config->module_search_path = buf; + return _Py_INIT_OK(); } -#define DECODE_LOCALE_ERR(NAME, LEN) \ - ((LEN) == (size_t)-2) \ - ? _Py_INIT_USER_ERR("failed to decode " #NAME) \ - : _Py_INIT_NO_MEMORY() - - static _PyInitError calculate_init(PyCalculatePath *calculate, const _PyMainInterpreterConfig *main_config) @@ -941,15 +954,30 @@ calculate_free(PyCalculatePath *calculate) } -static void +static _PyInitError calculate_path_impl(PyCalculatePath *calculate, PyPathConfig *config) { - calculate_progpath(calculate, config); - calculate_argv0_path(calculate, config); + _PyInitError err = calculate_program_name(calculate, config); + if (_Py_INIT_FAILED(err)) { + return err; + } + + err = calculate_argv0_path(calculate, config->program_name); + if (_Py_INIT_FAILED(err)) { + return err; + } + calculate_read_pyenv(calculate); - calculate_prefix(calculate, config); - calculate_zip_path(calculate, config); - calculate_exec_prefix(calculate, config); + + wchar_t prefix[MAXPATHLEN+1]; + memset(prefix, 0, sizeof(prefix)); + calculate_prefix(calculate, prefix); + + calculate_zip_path(calculate, prefix); + + wchar_t exec_prefix[MAXPATHLEN+1]; + memset(exec_prefix, 0, sizeof(exec_prefix)); + calculate_exec_prefix(calculate, exec_prefix); if ((!calculate->prefix_found || !calculate->exec_prefix_found) && !Py_FrozenFlag) @@ -958,53 +986,105 @@ calculate_path_impl(PyCalculatePath *calculate, PyPathConfig *config) "Consider setting $PYTHONHOME to [:]\n"); } - config->module_search_path = calculate_module_search_path(calculate, config); - calculate_reduce_prefix(calculate, config); - calculate_reduce_exec_prefix(calculate, config); + err = calculate_module_search_path(calculate, prefix, exec_prefix, + config); + if (_Py_INIT_FAILED(err)) { + return err; + } + + calculate_reduce_prefix(calculate, prefix); + + config->prefix = _PyMem_RawWcsdup(prefix); + if (config->prefix == NULL) { + return _Py_INIT_NO_MEMORY(); + } + + calculate_reduce_exec_prefix(calculate, exec_prefix); + + config->exec_prefix = _PyMem_RawWcsdup(exec_prefix); + if (config->exec_prefix == NULL) { + return _Py_INIT_NO_MEMORY(); + } + + return _Py_INIT_OK(); } static void -calculate_path(const _PyMainInterpreterConfig *main_config) +pathconfig_clear(PyPathConfig *config) { - _PyInitError err; - PyCalculatePath calculate; - memset(&calculate, 0, sizeof(calculate)); +#define CLEAR(ATTR) \ + do { \ + PyMem_RawFree(ATTR); \ + ATTR = NULL; \ + } while (0) + + CLEAR(config->prefix); + CLEAR(config->exec_prefix); + CLEAR(config->program_name); + CLEAR(config->module_search_path); +#undef CLEAR +} - _PyMainInterpreterConfig tmp_config; - int use_tmp = (main_config == NULL); - if (use_tmp) { - tmp_config = _PyMainInterpreterConfig_INIT; - err = _PyMainInterpreterConfig_ReadEnv(&tmp_config); - if (_Py_INIT_FAILED(err)) { - goto fatal_error; - } - main_config = &tmp_config; + +/* Initialize paths for Py_GetPath(), Py_GetPrefix(), Py_GetExecPrefix() + and Py_GetProgramFullPath() */ +_PyInitError +_PyPathConfig_Init(const _PyMainInterpreterConfig *main_config) +{ + if (path_config.module_search_path) { + /* Already initialized */ + return _Py_INIT_OK(); } - err = calculate_init(&calculate, main_config); + PyCalculatePath calculate; + memset(&calculate, 0, sizeof(calculate)); + + _PyInitError err = calculate_init(&calculate, main_config); if (_Py_INIT_FAILED(err)) { - goto fatal_error; + goto done; } PyPathConfig new_path_config; memset(&new_path_config, 0, sizeof(new_path_config)); - calculate_path_impl(&calculate, &new_path_config); + err = calculate_path_impl(&calculate, &new_path_config); + if (_Py_INIT_FAILED(err)) { + pathconfig_clear(&new_path_config); + goto done; + } + path_config = new_path_config; + err = _Py_INIT_OK(); - if (use_tmp) { - _PyMainInterpreterConfig_Clear(&tmp_config); - } +done: calculate_free(&calculate); - return; + return err; +} + + +static void +calculate_path(void) +{ + _PyInitError err; + _PyMainInterpreterConfig config = _PyMainInterpreterConfig_INIT; -fatal_error: - if (use_tmp) { - _PyMainInterpreterConfig_Clear(&tmp_config); + err = _PyMainInterpreterConfig_ReadEnv(&config); + if (!_Py_INIT_FAILED(err)) { + err = _PyPathConfig_Init(&config); } - calculate_free(&calculate); - _Py_FatalInitError(err); + _PyMainInterpreterConfig_Clear(&config); + + if (_Py_INIT_FAILED(err)) { + _Py_FatalInitError(err); + } +} + + +void +_PyPathConfig_Fini(void) +{ + pathconfig_clear(&path_config); } @@ -1012,33 +1092,19 @@ calculate_path(const _PyMainInterpreterConfig *main_config) void Py_SetPath(const wchar_t *path) { - if (path_config.module_search_path != NULL) { - PyMem_RawFree(path_config.module_search_path); - path_config.module_search_path = NULL; - } - if (path == NULL) { + pathconfig_clear(&path_config); return; } - wchar_t *program_name = Py_GetProgramName(); - wcsncpy(path_config.program_name, program_name, MAXPATHLEN); - path_config.exec_prefix[0] = path_config.prefix[0] = L'\0'; - size_t size = (wcslen(path) + 1) * sizeof(wchar_t); - path_config.module_search_path = PyMem_RawMalloc(size); - if (path_config.module_search_path != NULL) { - wcscpy(path_config.module_search_path, path); - } -} - + PyPathConfig new_config; + new_config.program_name = _PyMem_RawWcsdup(Py_GetProgramName()); + new_config.exec_prefix = _PyMem_RawWcsdup(L""); + new_config.prefix = _PyMem_RawWcsdup(L""); + new_config.module_search_path = _PyMem_RawWcsdup(path); -wchar_t * -_Py_GetPathWithConfig(const _PyMainInterpreterConfig *main_config) -{ - if (!path_config.module_search_path) { - calculate_path(main_config); - } - return path_config.module_search_path; + pathconfig_clear(&path_config); + path_config = new_config; } @@ -1046,7 +1112,7 @@ wchar_t * Py_GetPath(void) { if (!path_config.module_search_path) { - calculate_path(NULL); + calculate_path(); } return path_config.module_search_path; } @@ -1056,7 +1122,7 @@ wchar_t * Py_GetPrefix(void) { if (!path_config.module_search_path) { - calculate_path(NULL); + calculate_path(); } return path_config.prefix; } @@ -1066,7 +1132,7 @@ wchar_t * Py_GetExecPrefix(void) { if (!path_config.module_search_path) { - calculate_path(NULL); + calculate_path(); } return path_config.exec_prefix; } @@ -1076,7 +1142,7 @@ wchar_t * Py_GetProgramFullPath(void) { if (!path_config.module_search_path) { - calculate_path(NULL); + calculate_path(); } return path_config.program_name; } diff --git a/Modules/main.c b/Modules/main.c index dca165dad2c41a..1b1d8041ef114f 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -38,14 +38,14 @@ extern "C" { #define DECODE_LOCALE_ERR(NAME, LEN) \ (((LEN) == -2) \ - ? _Py_INIT_USER_ERR("failed to decode " #NAME) \ + ? _Py_INIT_USER_ERR("cannot decode " #NAME) \ : _Py_INIT_NO_MEMORY()) #define SET_DECODE_ERROR(NAME, LEN) \ do { \ if ((LEN) == (size_t)-2) { \ - pymain->err = _Py_INIT_USER_ERR("failed to decode " #NAME); \ + pymain->err = _Py_INIT_USER_ERR("cannot decode " #NAME); \ } \ else { \ pymain->err = _Py_INIT_NO_MEMORY(); \ diff --git a/PC/getpathp.c b/PC/getpathp.c index 38e433b6dcd05a..fe0226ba5d277c 100644 --- a/PC/getpathp.c +++ b/PC/getpathp.c @@ -117,9 +117,9 @@ #endif typedef struct { - wchar_t prefix[MAXPATHLEN+1]; - wchar_t program_name[MAXPATHLEN+1]; - wchar_t dllpath[MAXPATHLEN+1]; + wchar_t *prefix; + wchar_t *program_name; + wchar_t *dll_path; wchar_t *module_search_path; } PyPathConfig; @@ -483,24 +483,39 @@ getpythonregpath(HKEY keyBase, int skipcore) #endif /* Py_ENABLE_SHARED */ -static void -get_progpath(PyCalculatePath *calculate, PyPathConfig *config) +static _PyInitError +get_dll_path(PyCalculatePath *calculate, PyPathConfig *config) { - wchar_t *path = calculate->path_env; + wchar_t dll_path[MAXPATHLEN+1]; + memset(dll_path, 0, sizeof(dll_path)); #ifdef Py_ENABLE_SHARED extern HANDLE PyWin_DLLhModule; - /* static init of program_name ensures final char remains \0 */ if (PyWin_DLLhModule) { - if (!GetModuleFileNameW(PyWin_DLLhModule, config->dllpath, MAXPATHLEN)) { - config->dllpath[0] = 0; + if (!GetModuleFileNameW(PyWin_DLLhModule, dll_path, MAXPATHLEN)) { + dll_path[0] = 0; } } #else - config->dllpath[0] = 0; + dll_path[0] = 0; #endif - if (GetModuleFileNameW(NULL, config->program_name, MAXPATHLEN)) { - return; + + config->dll_path = _PyMem_RawWcsdup(dll_path); + if (config->dll_path == NULL) { + return _Py_INIT_NO_MEMORY(); + } + return _Py_INIT_OK(); +} + + +static _PyInitError +get_program_name(PyCalculatePath *calculate, PyPathConfig *config) +{ + wchar_t program_name[MAXPATHLEN+1]; + memset(program_name, 0, sizeof(program_name)); + + if (GetModuleFileNameW(NULL, program_name, MAXPATHLEN)) { + goto done; } /* If there is no slash in the argv0 path, then we have to @@ -514,9 +529,10 @@ get_progpath(PyCalculatePath *calculate, PyPathConfig *config) if (wcschr(calculate->program_name, SEP)) #endif { - wcsncpy(config->program_name, calculate->program_name, MAXPATHLEN); + wcsncpy(program_name, calculate->program_name, MAXPATHLEN); } - else if (path) { + else if (calculate->path_env) { + wchar_t *path = calculate->path_env; while (1) { wchar_t *delim = wcschr(path, DELIM); @@ -524,29 +540,36 @@ get_progpath(PyCalculatePath *calculate, PyPathConfig *config) size_t len = delim - path; /* ensure we can't overwrite buffer */ len = min(MAXPATHLEN,len); - wcsncpy(config->program_name, path, len); - *(config->program_name + len) = '\0'; + wcsncpy(program_name, path, len); + program_name[len] = '\0'; } else { - wcsncpy(config->program_name, path, MAXPATHLEN); + wcsncpy(program_name, path, MAXPATHLEN); } /* join() is safe for MAXPATHLEN+1 size buffer */ - join(config->program_name, calculate->program_name); - if (exists(config->program_name)) { + join(program_name, calculate->program_name); + if (exists(program_name)) { break; } if (!delim) { - config->program_name[0] = '\0'; + program_name[0] = '\0'; break; } path = delim + 1; } } else { - config->program_name[0] = '\0'; + program_name[0] = '\0'; + } + +done: + config->program_name = _PyMem_RawWcsdup(program_name); + if (config->program_name == NULL) { + return _Py_INIT_NO_MEMORY(); } + return _Py_INIT_OK(); } @@ -602,12 +625,13 @@ find_env_config_value(FILE * env_file, const wchar_t * key, wchar_t * value) } -static wchar_t* -read_pth_file(const wchar_t *path, wchar_t *prefix, int *isolated, int *nosite) +static int +read_pth_file(PyPathConfig *config, wchar_t *prefix, const wchar_t *path, + int *isolated, int *nosite) { FILE *sp_file = _Py_wfopen(path, L"r"); if (sp_file == NULL) { - return NULL; + return 0; } wcscpy_s(prefix, MAXPATHLEN+1, path); @@ -680,12 +704,13 @@ read_pth_file(const wchar_t *path, wchar_t *prefix, int *isolated, int *nosite) } fclose(sp_file); - return buf; + config->module_search_path = buf; + return 1; error: PyMem_RawFree(buf); fclose(sp_file); - return NULL; + return 0; } @@ -704,8 +729,8 @@ calculate_init(PyCalculatePath *calculate, static int get_pth_filename(wchar_t *spbuffer, PyPathConfig *config) { - if (config->dllpath[0]) { - if (!change_ext(spbuffer, config->dllpath, L"._pth") && exists(spbuffer)) { + if (config->dll_path[0]) { + if (!change_ext(spbuffer, config->dll_path, L"._pth") && exists(spbuffer)) { return 1; } } @@ -719,7 +744,7 @@ get_pth_filename(wchar_t *spbuffer, PyPathConfig *config) static int -calculate_pth_file(PyPathConfig *config) +calculate_pth_file(PyPathConfig *config, wchar_t *prefix) { wchar_t spbuffer[MAXPATHLEN+1]; @@ -727,13 +752,8 @@ calculate_pth_file(PyPathConfig *config) return 0; } - config->module_search_path = read_pth_file(spbuffer, config->prefix, - &Py_IsolatedFlag, - &Py_NoSiteFlag); - if (!config->module_search_path) { - return 0; - } - return 1; + return read_pth_file(config, prefix, spbuffer, + &Py_IsolatedFlag, &Py_NoSiteFlag); } @@ -775,43 +795,34 @@ calculate_pyvenv_file(PyCalculatePath *calculate) } -static void -calculate_path_impl(PyCalculatePath *calculate, PyPathConfig *config, - const _PyMainInterpreterConfig *main_config) -{ - get_progpath(calculate, config); - /* program_name guaranteed \0 terminated in MAXPATH+1 bytes. */ - wcscpy_s(calculate->argv0_path, MAXPATHLEN+1, config->program_name); - reduce(calculate->argv0_path); +#define INIT_ERR_BUFFER_OVERFLOW() _Py_INIT_ERR("buffer overflow") - /* Search for a sys.path file */ - if (calculate_pth_file(config)) { - return; - } - - calculate_pyvenv_file(calculate); - - /* Calculate zip archive path from DLL or exe path */ - change_ext(calculate->zip_path, - config->dllpath[0] ? config->dllpath : config->program_name, - L".zip"); +static void +calculate_home_prefix(PyCalculatePath *calculate, wchar_t *prefix) +{ if (calculate->home == NULL || *calculate->home == '\0') { if (calculate->zip_path[0] && exists(calculate->zip_path)) { - wcscpy_s(config->prefix, MAXPATHLEN+1, calculate->zip_path); - reduce(config->prefix); - calculate->home = config->prefix; - } else if (search_for_prefix(config->prefix, calculate->argv0_path, LANDMARK)) { - calculate->home = config->prefix; + wcscpy_s(prefix, MAXPATHLEN+1, calculate->zip_path); + reduce(prefix); + calculate->home = prefix; + } + else if (search_for_prefix(prefix, calculate->argv0_path, LANDMARK)) { + calculate->home = prefix; } else { calculate->home = NULL; } } else { - wcscpy_s(config->prefix, MAXPATHLEN+1, calculate->home); + wcscpy_s(prefix, MAXPATHLEN+1, calculate->home); } +} + +static _PyInitError +calculate_module_search_path(PyCalculatePath *calculate, PyPathConfig *config, wchar_t *prefix) +{ int skiphome = calculate->home==NULL ? 0 : 1; #ifdef Py_ENABLE_SHARED calculate->machine_path = getpythonregpath(HKEY_LOCAL_MACHINE, skiphome); @@ -873,34 +884,34 @@ calculate_path_impl(PyCalculatePath *calculate, PyPathConfig *config, fprintf(stderr, "Using default static path.\n"); config->module_search_path = PYTHONPATH; } - return; + return _Py_INIT_OK(); } start_buf = buf; if (calculate->module_search_path_env) { if (wcscpy_s(buf, bufsz - (buf - start_buf), calculate->module_search_path_env)) { - Py_FatalError("buffer overflow in getpathp.c's calculate_path()"); + return INIT_ERR_BUFFER_OVERFLOW(); } buf = wcschr(buf, L'\0'); *buf++ = DELIM; } if (calculate->zip_path[0]) { if (wcscpy_s(buf, bufsz - (buf - start_buf), calculate->zip_path)) { - Py_FatalError("buffer overflow in getpathp.c's calculate_path()"); + return INIT_ERR_BUFFER_OVERFLOW(); } buf = wcschr(buf, L'\0'); *buf++ = DELIM; } if (calculate->user_path) { if (wcscpy_s(buf, bufsz - (buf - start_buf), calculate->user_path)) { - Py_FatalError("buffer overflow in getpathp.c's calculate_path()"); + return INIT_ERR_BUFFER_OVERFLOW(); } buf = wcschr(buf, L'\0'); *buf++ = DELIM; } if (calculate->machine_path) { if (wcscpy_s(buf, bufsz - (buf - start_buf), calculate->machine_path)) { - Py_FatalError("buffer overflow in getpathp.c's calculate_path()"); + return INIT_ERR_BUFFER_OVERFLOW(); } buf = wcschr(buf, L'\0'); *buf++ = DELIM; @@ -908,7 +919,7 @@ calculate_path_impl(PyCalculatePath *calculate, PyPathConfig *config, if (calculate->home == NULL) { if (!skipdefault) { if (wcscpy_s(buf, bufsz - (buf - start_buf), PYTHONPATH)) { - Py_FatalError("buffer overflow in getpathp.c's calculate_path()"); + return INIT_ERR_BUFFER_OVERFLOW(); } buf = wcschr(buf, L'\0'); *buf++ = DELIM; @@ -927,7 +938,7 @@ calculate_path_impl(PyCalculatePath *calculate, PyPathConfig *config, } if (p[0] == '.' && is_sep(p[1])) { if (wcscpy_s(buf, bufsz - (buf - start_buf), calculate->home)) { - Py_FatalError("buffer overflow in getpathp.c's calculate_path()"); + return INIT_ERR_BUFFER_OVERFLOW(); } buf = wcschr(buf, L'\0'); p++; @@ -957,7 +968,7 @@ calculate_path_impl(PyCalculatePath *calculate, PyPathConfig *config, on the path, and that our 'prefix' directory is the parent of that. */ - if (config->prefix[0] == L'\0') { + if (prefix[0] == L'\0') { wchar_t lookBuf[MAXPATHLEN+1]; wchar_t *look = buf - 1; /* 'buf' is at the end of the buffer */ while (1) { @@ -974,7 +985,7 @@ calculate_path_impl(PyCalculatePath *calculate, PyPathConfig *config, lookBuf[nchars] = L'\0'; /* Up one level to the parent */ reduce(lookBuf); - if (search_for_prefix(config->prefix, lookBuf, LANDMARK)) { + if (search_for_prefix(prefix, lookBuf, LANDMARK)) { break; } /* If we are out of paths to search - give up */ @@ -986,6 +997,59 @@ calculate_path_impl(PyCalculatePath *calculate, PyPathConfig *config, } config->module_search_path = start_buf; + return _Py_INIT_OK(); +} + + +static _PyInitError +calculate_path_impl(PyCalculatePath *calculate, PyPathConfig *config, + const _PyMainInterpreterConfig *main_config) +{ + _PyInitError err; + + err = get_dll_path(calculate, config); + if (_Py_INIT_FAILED(err)) { + return err; + } + + err = get_program_name(calculate, config); + if (_Py_INIT_FAILED(err)) { + return err; + } + + /* program_name guaranteed \0 terminated in MAXPATH+1 bytes. */ + wcscpy_s(calculate->argv0_path, MAXPATHLEN+1, config->program_name); + reduce(calculate->argv0_path); + + wchar_t prefix[MAXPATHLEN+1]; + memset(prefix, 0, sizeof(prefix)); + + /* Search for a sys.path file */ + if (calculate_pth_file(config, prefix)) { + goto done; + } + + calculate_pyvenv_file(calculate); + + /* Calculate zip archive path from DLL or exe path */ + change_ext(calculate->zip_path, + config->dll_path[0] ? config->dll_path : config->program_name, + L".zip"); + + calculate_home_prefix(calculate, prefix); + + err = calculate_module_search_path(calculate, config, prefix); + if (_Py_INIT_FAILED(err)) { + return err; + } + +done: + config->prefix = _PyMem_RawWcsdup(prefix); + if (config->prefix == NULL) { + return _Py_INIT_NO_MEMORY(); + } + + return _Py_INIT_OK(); } @@ -996,47 +1060,85 @@ calculate_free(PyCalculatePath *calculate) PyMem_RawFree(calculate->user_path); } + static void -calculate_path(const _PyMainInterpreterConfig *main_config) +pathconfig_clear(PyPathConfig *config) { +#define CLEAR(ATTR) \ + do { \ + PyMem_RawFree(ATTR); \ + ATTR = NULL; \ + } while (0) + + CLEAR(config->prefix); + CLEAR(config->program_name); + CLEAR(config->dll_path); + CLEAR(config->module_search_path); +#undef CLEAR +} + + +/* Initialize paths for Py_GetPath(), Py_GetPrefix(), Py_GetExecPrefix() + and Py_GetProgramFullPath() */ +_PyInitError +_PyPathConfig_Init(const _PyMainInterpreterConfig *main_config) +{ + if (path_config.module_search_path) { + /* Already initialized */ + return _Py_INIT_OK(); + } + _PyInitError err; + PyCalculatePath calculate; memset(&calculate, 0, sizeof(calculate)); - _PyMainInterpreterConfig tmp_config; - int use_tmp = (main_config == NULL); - if (use_tmp) { - tmp_config = _PyMainInterpreterConfig_INIT; - err = _PyMainInterpreterConfig_ReadEnv(&tmp_config); - if (_Py_INIT_FAILED(err)) { - goto fatal_error; - } - main_config = &tmp_config; - } - calculate_init(&calculate, main_config); PyPathConfig new_path_config; memset(&new_path_config, 0, sizeof(new_path_config)); - calculate_path_impl(&calculate, &new_path_config, main_config); + err = calculate_path_impl(&calculate, &new_path_config, main_config); + if (_Py_INIT_FAILED(err)) { + goto done; + } + path_config = new_path_config; + err = _Py_INIT_OK(); - if (use_tmp) { - _PyMainInterpreterConfig_Clear(&tmp_config); +done: + if (_Py_INIT_FAILED(err)) { + pathconfig_clear(&new_path_config); } calculate_free(&calculate); - return; + return err; +} -fatal_error: - if (use_tmp) { - _PyMainInterpreterConfig_Clear(&tmp_config); + +static void +calculate_path(void) +{ + _PyInitError err; + _PyMainInterpreterConfig config = _PyMainInterpreterConfig_INIT; + + err = _PyMainInterpreterConfig_ReadEnv(&config); + if (!_Py_INIT_FAILED(err)) { + err = _PyPathConfig_Init(&config); + } + _PyMainInterpreterConfig_Clear(&config); + + if (_Py_INIT_FAILED(err)) { + _Py_FatalInitError(err); } - calculate_free(&calculate); - _Py_FatalInitError(err); } +void +_PyPathConfig_Fini(void) +{ + pathconfig_clear(&path_config); +} + /* External interface */ @@ -1044,31 +1146,21 @@ void Py_SetPath(const wchar_t *path) { if (path_config.module_search_path != NULL) { - PyMem_RawFree(path_config.module_search_path); - path_config.module_search_path = NULL; + pathconfig_clear(&path_config); } if (path == NULL) { return; } - wchar_t *program_name = Py_GetProgramName(); - wcsncpy(path_config.program_name, program_name, MAXPATHLEN); - path_config.prefix[0] = L'\0'; - path_config.module_search_path = PyMem_RawMalloc((wcslen(path) + 1) * sizeof(wchar_t)); - if (path_config.module_search_path != NULL) { - wcscpy(path_config.module_search_path, path); - } -} - + PyPathConfig new_config; + new_config.program_name = _PyMem_RawWcsdup(Py_GetProgramName()); + new_config.prefix = _PyMem_RawWcsdup(L""); + new_config.dll_path = _PyMem_RawWcsdup(L""); + new_config.module_search_path = _PyMem_RawWcsdup(path); -wchar_t * -_Py_GetPathWithConfig(const _PyMainInterpreterConfig *main_config) -{ - if (!path_config.module_search_path) { - calculate_path(main_config); - } - return path_config.module_search_path; + pathconfig_clear(&path_config); + path_config = new_config; } @@ -1076,7 +1168,7 @@ wchar_t * Py_GetPath(void) { if (!path_config.module_search_path) { - calculate_path(NULL); + calculate_path(); } return path_config.module_search_path; } @@ -1086,7 +1178,7 @@ wchar_t * Py_GetPrefix(void) { if (!path_config.module_search_path) { - calculate_path(NULL); + calculate_path(); } return path_config.prefix; } @@ -1103,7 +1195,7 @@ wchar_t * Py_GetProgramFullPath(void) { if (!path_config.module_search_path) { - calculate_path(NULL); + calculate_path(); } return path_config.program_name; } @@ -1129,7 +1221,7 @@ _Py_CheckPython3() /* If there is a python3.dll next to the python3y.dll, assume this is a build tree; use that DLL */ - wcscpy(py3path, path_config.dllpath); + wcscpy(py3path, path_config.dll_path); s = wcsrchr(py3path, L'\\'); if (!s) { s = py3path; diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 714be3768f6ba3..868ac8450d453c 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -866,11 +866,6 @@ _Py_InitializeMainInterpreter(const _PyMainInterpreterConfig *config) /* Now finish configuring the main interpreter */ interp->config = *config; - /* GetPath may initialize state that _PySys_EndInit locks - in, and so has to be called first. */ - /* TODO: Call Py_GetPath() in Py_ReadConfig, rather than here */ - wchar_t *sys_path = _Py_GetPathWithConfig(&interp->config); - if (interp->core_config._disable_importlib) { /* Special mode for freeze_importlib: run with no import system * @@ -880,11 +875,20 @@ _Py_InitializeMainInterpreter(const _PyMainInterpreterConfig *config) _PyRuntime.initialized = 1; return _Py_INIT_OK(); } + /* TODO: Report exceptions rather than fatal errors below here */ if (_PyTime_Init() < 0) return _Py_INIT_ERR("can't initialize time"); + /* GetPath may initialize state that _PySys_EndInit locks + in, and so has to be called first. */ + err = _PyPathConfig_Init(&interp->config); + if (_Py_INIT_FAILED(err)) { + return err; + } + wchar_t *sys_path = Py_GetPath(); + /* Finish setting up the sys module and import system */ PySys_SetPath(sys_path); if (_PySys_EndInit(interp->sysdict) < 0) @@ -1261,6 +1265,9 @@ Py_FinalizeEx(void) #endif call_ll_exitfuncs(); + + _PyPathConfig_Fini(); + _PyRuntime_Finalize(); return status; } @@ -1290,6 +1297,7 @@ new_interpreter(PyThreadState **tstate_p) PyInterpreterState *interp; PyThreadState *tstate, *save_tstate; PyObject *bimod, *sysmod; + _PyInitError err; if (!_PyRuntime.initialized) { return _Py_INIT_ERR("Py_Initialize must be called first"); @@ -1325,10 +1333,13 @@ new_interpreter(PyThreadState **tstate_p) interp->config = main_interp->config; } - /* XXX The following is lax in error checking */ - - wchar_t *sys_path = _Py_GetPathWithConfig(&interp->config); + err = _PyPathConfig_Init(&interp->config); + if (_Py_INIT_FAILED(err)) { + return err; + } + wchar_t *sys_path = Py_GetPath(); + /* XXX The following is lax in error checking */ PyObject *modules = PyDict_New(); if (modules == NULL) { return _Py_INIT_ERR("can't make modules dictionary"); @@ -1359,7 +1370,6 @@ new_interpreter(PyThreadState **tstate_p) if (bimod != NULL && sysmod != NULL) { PyObject *pstderr; - _PyInitError err; /* Set up a preliminary stderr printer until we have enough infrastructure for the io module in place. */ From 610e5afdcbe3eca906ef32f4e0364e20e1b1ad23 Mon Sep 17 00:00:00 2001 From: Mandeep Bhutani Date: Fri, 24 Nov 2017 22:56:00 -0600 Subject: [PATCH 40/75] bpo-30004: Fix the code example of using group in Regex Howto Docs (GH-4443) The provided code example was supposed to find repeated words, however it returned false results. --- Doc/howto/regex.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/howto/regex.rst b/Doc/howto/regex.rst index e8466ee5423c68..fa8c6939408100 100644 --- a/Doc/howto/regex.rst +++ b/Doc/howto/regex.rst @@ -844,7 +844,7 @@ backreferences in a RE. For example, the following RE detects doubled words in a string. :: - >>> p = re.compile(r'(\b\w+)\s+\1') + >>> p = re.compile(r'\b(\w+)\s+\1\b') >>> p.search('Paris in the the spring').group() 'the the' @@ -943,9 +943,9 @@ number of the group. There's naturally a variant that uses the group name instead of the number. This is another Python extension: ``(?P=name)`` indicates that the contents of the group called *name* should again be matched at the current point. The regular expression for finding doubled words, -``(\b\w+)\s+\1`` can also be written as ``(?P\b\w+)\s+(?P=word)``:: +``\b(\w+)\s+\1\b`` can also be written as ``\b(?P\w+)\s+(?P=word)\b``:: - >>> p = re.compile(r'(?P\b\w+)\s+(?P=word)') + >>> p = re.compile(r'\b(?P\w+)\s+(?P=word)\b') >>> p.search('Paris in the the spring').group() 'the the' From 9d5ec808de2c1359f434cc2fa8378458e4339c96 Mon Sep 17 00:00:00 2001 From: Mariatta Date: Fri, 24 Nov 2017 21:43:01 -0800 Subject: [PATCH 41/75] Improve Scheduler Objects documentation. (GH-4556) Mention that the lower the priority number, the higher priority it represents. --- Doc/library/sched.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/sched.rst b/Doc/library/sched.rst index 4d4a6161057cc5..6094a7b871454e 100644 --- a/Doc/library/sched.rst +++ b/Doc/library/sched.rst @@ -69,7 +69,7 @@ Scheduler Objects Schedule a new event. The *time* argument should be a numeric type compatible with the return value of the *timefunc* function passed to the constructor. Events scheduled for the same *time* will be executed in the order of their - *priority*. + *priority*. A lower number represents a higher priority. Executing the event means executing ``action(*argument, **kwargs)``. *argument* is a sequence holding the positional arguments for *action*. From 8d9bb11d8fcbf10cc9b1eb0a647bcf3658a4e3dd Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Sat, 25 Nov 2017 13:37:22 +0300 Subject: [PATCH 42/75] bpo-28334: netrc() now uses expanduser() to find .netrc file (GH-4537) Previously, netrc.netrc() was raised an exception if $HOME is not set. Authored-By: Dimitri Merejkowsky --- Doc/library/netrc.rst | 11 +++-- Lib/netrc.py | 5 +-- Lib/test/test_netrc.py | 43 +++++++++++++++++-- Misc/ACKS | 1 + .../2017-11-24-11-50-41.bpo-28334.3gGGlt.rst | 3 ++ 5 files changed, 53 insertions(+), 10 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2017-11-24-11-50-41.bpo-28334.3gGGlt.rst diff --git a/Doc/library/netrc.rst b/Doc/library/netrc.rst index 64aa3ac7c8aefd..3d29ac49b9191a 100644 --- a/Doc/library/netrc.rst +++ b/Doc/library/netrc.rst @@ -20,8 +20,10 @@ the Unix :program:`ftp` program and other FTP clients. A :class:`~netrc.netrc` instance or subclass instance encapsulates data from a netrc file. The initialization argument, if present, specifies the file to parse. If - no argument is given, the file :file:`.netrc` in the user's home directory will - be read. Parse errors will raise :exc:`NetrcParseError` with diagnostic + no argument is given, the file :file:`.netrc` in the user's home directory -- + as determined by :func:`os.path.expanduser` -- will be read. Otherwise, + a :exc:`FileNotFoundError` exception will be raised. + Parse errors will raise :exc:`NetrcParseError` with diagnostic information including the file name, line number, and terminating token. If no argument is specified on a POSIX system, the presence of passwords in the :file:`.netrc` file will raise a :exc:`NetrcParseError` if the file @@ -32,6 +34,10 @@ the Unix :program:`ftp` program and other FTP clients. .. versionchanged:: 3.4 Added the POSIX permission check. + .. versionchanged:: 3.7 + :func:`os.path.expanduser` is used to find the location of the + :file:`.netrc` file when *file* is not passed as argument. + .. exception:: NetrcParseError @@ -82,4 +88,3 @@ Instances of :class:`~netrc.netrc` have public instance variables: punctuation is allowed in passwords, however, note that whitespace and non-printable characters are not allowed in passwords. This is a limitation of the way the .netrc file is parsed and may be removed in the future. - diff --git a/Lib/netrc.py b/Lib/netrc.py index baf8f1d99d9d30..f0ae48cfed9e67 100644 --- a/Lib/netrc.py +++ b/Lib/netrc.py @@ -23,10 +23,7 @@ class netrc: def __init__(self, file=None): default_netrc = file is None if file is None: - try: - file = os.path.join(os.environ['HOME'], ".netrc") - except KeyError: - raise OSError("Could not find .netrc: $HOME is not set") from None + file = os.path.join(os.path.expanduser("~"), ".netrc") self.hosts = {} self.macros = {} with open(file) as fp: diff --git a/Lib/test/test_netrc.py b/Lib/test/test_netrc.py index ca6f27d03c3afd..f59e5371acad3b 100644 --- a/Lib/test/test_netrc.py +++ b/Lib/test/test_netrc.py @@ -1,4 +1,5 @@ import netrc, os, unittest, sys, tempfile, textwrap +from unittest import mock from test import support @@ -126,8 +127,44 @@ def test_security(self): os.chmod(fn, 0o622) self.assertRaises(netrc.NetrcParseError, netrc.netrc) -def test_main(): - support.run_unittest(NetrcTestCase) + def test_file_not_found_in_home(self): + d = support.TESTFN + os.mkdir(d) + self.addCleanup(support.rmtree, d) + with support.EnvironmentVarGuard() as environ: + environ.set('HOME', d) + self.assertRaises(FileNotFoundError, netrc.netrc) + + def test_file_not_found_explicit(self): + self.assertRaises(FileNotFoundError, netrc.netrc, + file='unlikely_netrc') + + def test_home_not_set(self): + fake_home = support.TESTFN + os.mkdir(fake_home) + self.addCleanup(support.rmtree, fake_home) + fake_netrc_path = os.path.join(fake_home, '.netrc') + with open(fake_netrc_path, 'w') as f: + f.write('machine foo.domain.com login bar password pass') + os.chmod(fake_netrc_path, 0o600) + + orig_expanduser = os.path.expanduser + called = [] + + def fake_expanduser(s): + called.append(s) + with support.EnvironmentVarGuard() as environ: + environ.set('HOME', fake_home) + result = orig_expanduser(s) + return result + + with support.swap_attr(os.path, 'expanduser', fake_expanduser): + nrc = netrc.netrc() + login, account, password = nrc.authenticators('foo.domain.com') + self.assertEqual(login, 'bar') + + self.assertTrue(called) + if __name__ == "__main__": - test_main() + unittest.main() diff --git a/Misc/ACKS b/Misc/ACKS index 05113903f06de7..fc7154f762ecb4 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1031,6 +1031,7 @@ Bill van Melle Lucas Prado Melo Ezio Melotti Doug Mennella +Dimitri Merejkowsky Brian Merrell Alexis Métaireau Luke Mewburn diff --git a/Misc/NEWS.d/next/Library/2017-11-24-11-50-41.bpo-28334.3gGGlt.rst b/Misc/NEWS.d/next/Library/2017-11-24-11-50-41.bpo-28334.3gGGlt.rst new file mode 100644 index 00000000000000..074036b1d31e36 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-11-24-11-50-41.bpo-28334.3gGGlt.rst @@ -0,0 +1,3 @@ +Use :func:`os.path.expanduser` to find the ``~/.netrc`` file in +:class:`netrc.netrc`. If it does not exist, :exc:`FileNotFoundError` +is raised. Patch by Dimitri Merejkowsky. From fb2c2f2ee3847c9e74d87a32876f26f0e1ce3304 Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Sat, 25 Nov 2017 10:10:12 -0500 Subject: [PATCH 43/75] DEBUGGING TRAVIS - DO NOT MERGE --- Lib/uuid.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Lib/uuid.py b/Lib/uuid.py index 9e7c672f7a0a6f..c551bb60a0c9bd 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -374,7 +374,9 @@ def _find_mac(command, args, hw_identifiers, get_index): word = words[get_index(i)] mac = int(word.replace(b':', b''), 16) if _is_universal(mac): + print('%012x' % mac, 'universal', command) return mac + print('%012x' % mac, 'local', command) first_local_mac = first_local_mac or mac except (ValueError, IndexError): # Virtual interfaces, such as those provided by From 5b48dc638b7405fd9bde4d854bf477dfeaaddf44 Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Sat, 25 Nov 2017 16:23:52 +0100 Subject: [PATCH 44/75] bpo-32071: Add unittest -k option (#4496) * bpo-32071: Add unittest -k option --- Doc/library/unittest.rst | 34 +++++++++++++++++++ Lib/unittest/loader.py | 24 ++++++++----- Lib/unittest/main.py | 25 +++++++++++--- Lib/unittest/test/test_loader.py | 27 +++++++++++++++ Lib/unittest/test/test_program.py | 28 +++++++++++++++ .../2017-11-22-19-52-17.bpo-32071.4WNhUH.rst | 2 ++ 6 files changed, 126 insertions(+), 14 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2017-11-22-19-52-17.bpo-32071.4WNhUH.rst diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst index e52f140029c57a..4755488d91db06 100644 --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -219,6 +219,22 @@ Command-line options Stop the test run on the first error or failure. +.. cmdoption:: -k + + Only run test methods and classes that match the pattern or substring. + This option may be used multiple times, in which case all test cases that + match of the given patterns are included. + + Patterns that contain a wildcard character (``*``) are matched against the + test name using :meth:`fnmatch.fnmatchcase`; otherwise simple case-sensitive + substring matching is used. + + Patterns are matched against the fully qualified test method name as + imported by the test loader. + + For example, ``-k foo`` matches ``foo_tests.SomeTest.test_something``, + ``bar_tests.SomeTest.test_foo``, but not ``bar_tests.FooTest.test_something``. + .. cmdoption:: --locals Show local variables in tracebacks. @@ -229,6 +245,9 @@ Command-line options .. versionadded:: 3.5 The command-line option ``--locals``. +.. versionadded:: 3.7 + The command-line option ``-k``. + The command line can also be used for test discovery, for running all of the tests in a project or just a subset. @@ -1745,6 +1764,21 @@ Loading and running tests This affects all the :meth:`loadTestsFrom\*` methods. + .. attribute:: testNamePatterns + + List of Unix shell-style wildcard test name patterns that test methods + have to match to be included in test suites (see ``-v`` option). + + If this attribute is not ``None`` (the default), all test methods to be + included in test suites must match one of the patterns in this list. + Note that matches are always performed using :meth:`fnmatch.fnmatchcase`, + so unlike patterns passed to the ``-v`` option, simple substring patterns + will have to be converted using ``*`` wildcards. + + This affects all the :meth:`loadTestsFrom\*` methods. + + .. versionadded:: 3.7 + .. class:: TestResult diff --git a/Lib/unittest/loader.py b/Lib/unittest/loader.py index e860debc0f3920..eb03b4ab87707a 100644 --- a/Lib/unittest/loader.py +++ b/Lib/unittest/loader.py @@ -8,7 +8,7 @@ import functools import warnings -from fnmatch import fnmatch +from fnmatch import fnmatch, fnmatchcase from . import case, suite, util @@ -70,6 +70,7 @@ class TestLoader(object): """ testMethodPrefix = 'test' sortTestMethodsUsing = staticmethod(util.three_way_cmp) + testNamePatterns = None suiteClass = suite.TestSuite _top_level_dir = None @@ -222,11 +223,15 @@ def loadTestsFromNames(self, names, module=None): def getTestCaseNames(self, testCaseClass): """Return a sorted sequence of method names found within testCaseClass """ - def isTestMethod(attrname, testCaseClass=testCaseClass, - prefix=self.testMethodPrefix): - return attrname.startswith(prefix) and \ - callable(getattr(testCaseClass, attrname)) - testFnNames = list(filter(isTestMethod, dir(testCaseClass))) + def shouldIncludeMethod(attrname): + testFunc = getattr(testCaseClass, attrname) + isTestMethod = attrname.startswith(self.testMethodPrefix) and callable(testFunc) + if not isTestMethod: + return False + fullName = '%s.%s' % (testCaseClass.__module__, testFunc.__qualname__) + return self.testNamePatterns is None or \ + any(fnmatchcase(fullName, pattern) for pattern in self.testNamePatterns) + testFnNames = list(filter(shouldIncludeMethod, dir(testCaseClass))) if self.sortTestMethodsUsing: testFnNames.sort(key=functools.cmp_to_key(self.sortTestMethodsUsing)) return testFnNames @@ -486,16 +491,17 @@ def _find_test_path(self, full_path, pattern, namespace=False): defaultTestLoader = TestLoader() -def _makeLoader(prefix, sortUsing, suiteClass=None): +def _makeLoader(prefix, sortUsing, suiteClass=None, testNamePatterns=None): loader = TestLoader() loader.sortTestMethodsUsing = sortUsing loader.testMethodPrefix = prefix + loader.testNamePatterns = testNamePatterns if suiteClass: loader.suiteClass = suiteClass return loader -def getTestCaseNames(testCaseClass, prefix, sortUsing=util.three_way_cmp): - return _makeLoader(prefix, sortUsing).getTestCaseNames(testCaseClass) +def getTestCaseNames(testCaseClass, prefix, sortUsing=util.three_way_cmp, testNamePatterns=None): + return _makeLoader(prefix, sortUsing, testNamePatterns=testNamePatterns).getTestCaseNames(testCaseClass) def makeSuite(testCaseClass, prefix='test', sortUsing=util.three_way_cmp, suiteClass=suite.TestSuite): diff --git a/Lib/unittest/main.py b/Lib/unittest/main.py index 807604f08dfd14..e62469aa2a170f 100644 --- a/Lib/unittest/main.py +++ b/Lib/unittest/main.py @@ -46,6 +46,12 @@ def _convert_names(names): return [_convert_name(name) for name in names] +def _convert_select_pattern(pattern): + if not '*' in pattern: + pattern = '*%s*' % pattern + return pattern + + class TestProgram(object): """A command-line program that runs a set of tests; this is primarily for making test modules conveniently executable. @@ -53,7 +59,7 @@ class TestProgram(object): # defaults for testing module=None verbosity = 1 - failfast = catchbreak = buffer = progName = warnings = None + failfast = catchbreak = buffer = progName = warnings = testNamePatterns = None _discovery_parser = None def __init__(self, module='__main__', defaultTest=None, argv=None, @@ -140,8 +146,13 @@ def parseArgs(self, argv): self.testNames = list(self.defaultTest) self.createTests() - def createTests(self): - if self.testNames is None: + def createTests(self, from_discovery=False, Loader=None): + if self.testNamePatterns: + self.testLoader.testNamePatterns = self.testNamePatterns + if from_discovery: + loader = self.testLoader if Loader is None else Loader() + self.test = loader.discover(self.start, self.pattern, self.top) + elif self.testNames is None: self.test = self.testLoader.loadTestsFromModule(self.module) else: self.test = self.testLoader.loadTestsFromNames(self.testNames, @@ -179,6 +190,11 @@ def _getParentArgParser(self): action='store_true', help='Buffer stdout and stderr during tests') self.buffer = False + if self.testNamePatterns is None: + parser.add_argument('-k', dest='testNamePatterns', + action='append', type=_convert_select_pattern, + help='Only run tests which match the given substring') + self.testNamePatterns = [] return parser @@ -225,8 +241,7 @@ def _do_discovery(self, argv, Loader=None): self._initArgParsers() self._discovery_parser.parse_args(argv, self) - loader = self.testLoader if Loader is None else Loader() - self.test = loader.discover(self.start, self.pattern, self.top) + self.createTests(from_discovery=True, Loader=Loader) def runTests(self): if self.catchbreak: diff --git a/Lib/unittest/test/test_loader.py b/Lib/unittest/test/test_loader.py index 1131a755eaa3f2..15b01863f51454 100644 --- a/Lib/unittest/test/test_loader.py +++ b/Lib/unittest/test/test_loader.py @@ -1226,6 +1226,33 @@ def test_3(self): pass names = ['test_1', 'test_2', 'test_3'] self.assertEqual(loader.getTestCaseNames(TestC), names) + # "Return a sorted sequence of method names found within testCaseClass" + # + # If TestLoader.testNamePatterns is set, only tests that match one of these + # patterns should be included. + def test_getTestCaseNames__testNamePatterns(self): + class MyTest(unittest.TestCase): + def test_1(self): pass + def test_2(self): pass + def foobar(self): pass + + loader = unittest.TestLoader() + + loader.testNamePatterns = [] + self.assertEqual(loader.getTestCaseNames(MyTest), []) + + loader.testNamePatterns = ['*1'] + self.assertEqual(loader.getTestCaseNames(MyTest), ['test_1']) + + loader.testNamePatterns = ['*1', '*2'] + self.assertEqual(loader.getTestCaseNames(MyTest), ['test_1', 'test_2']) + + loader.testNamePatterns = ['*My*'] + self.assertEqual(loader.getTestCaseNames(MyTest), ['test_1', 'test_2']) + + loader.testNamePatterns = ['*my*'] + self.assertEqual(loader.getTestCaseNames(MyTest), []) + ################################################################ ### /Tests for TestLoader.getTestCaseNames() diff --git a/Lib/unittest/test/test_program.py b/Lib/unittest/test/test_program.py index 1cfc17959e074a..4a62ae1b11306e 100644 --- a/Lib/unittest/test/test_program.py +++ b/Lib/unittest/test/test_program.py @@ -2,6 +2,7 @@ import os import sys +import subprocess from test import support import unittest import unittest.test @@ -409,6 +410,33 @@ def testParseArgsAbsolutePathsThatCannotBeConverted(self): # for invalid filenames should we raise a useful error rather than # leaving the current error message (import of filename fails) in place? + def testParseArgsSelectedTestNames(self): + program = self.program + argv = ['progname', '-k', 'foo', '-k', 'bar', '-k', '*pat*'] + + program.createTests = lambda: None + program.parseArgs(argv) + + self.assertEqual(program.testNamePatterns, ['*foo*', '*bar*', '*pat*']) + + def testSelectedTestNamesFunctionalTest(self): + def run_unittest(args): + p = subprocess.Popen([sys.executable, '-m', 'unittest'] + args, + stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, cwd=os.path.dirname(__file__)) + with p: + _, stderr = p.communicate() + return stderr.decode() + + t = '_test_warnings' + self.assertIn('Ran 7 tests', run_unittest([t])) + self.assertIn('Ran 7 tests', run_unittest(['-k', 'TestWarnings', t])) + self.assertIn('Ran 7 tests', run_unittest(['discover', '-p', '*_test*', '-k', 'TestWarnings'])) + self.assertIn('Ran 2 tests', run_unittest(['-k', 'f', t])) + self.assertIn('Ran 7 tests', run_unittest(['-k', 't', t])) + self.assertIn('Ran 3 tests', run_unittest(['-k', '*t', t])) + self.assertIn('Ran 7 tests', run_unittest(['-k', '*test_warnings.*Warning*', t])) + self.assertIn('Ran 1 test', run_unittest(['-k', '*test_warnings.*warning*', t])) + if __name__ == '__main__': unittest.main() diff --git a/Misc/NEWS.d/next/Library/2017-11-22-19-52-17.bpo-32071.4WNhUH.rst b/Misc/NEWS.d/next/Library/2017-11-22-19-52-17.bpo-32071.4WNhUH.rst new file mode 100644 index 00000000000000..2f0f54041cbf6a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-11-22-19-52-17.bpo-32071.4WNhUH.rst @@ -0,0 +1,2 @@ +Added the ``-k`` command-line option to ``python -m unittest`` to run only +tests that match the given pattern(s). From cfa797c0681b7fef47cf93955fd06b54ddd09a7f Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 25 Nov 2017 17:38:20 +0200 Subject: [PATCH 45/75] bpo-24641: Improved error message for JSON unserializible keys. (#4364) Also updated an example for default() in the module docstring. Removed quotes around type name in other error messages. --- Lib/json/__init__.py | 7 ++++--- Lib/json/encoder.py | 7 ++++--- Lib/test/test_json/test_fail.py | 13 ++++++++----- Modules/_json.c | 5 +++-- 4 files changed, 19 insertions(+), 13 deletions(-) diff --git a/Lib/json/__init__.py b/Lib/json/__init__.py index 6d0511ebfe90cf..a5660099af7514 100644 --- a/Lib/json/__init__.py +++ b/Lib/json/__init__.py @@ -76,7 +76,8 @@ >>> def encode_complex(obj): ... if isinstance(obj, complex): ... return [obj.real, obj.imag] - ... raise TypeError(repr(obj) + " is not JSON serializable") + ... raise TypeError(f'Object of type {obj.__class__.__name__} ' + ... f'is not JSON serializable') ... >>> json.dumps(2 + 1j, default=encode_complex) '[2.0, 1.0]' @@ -344,8 +345,8 @@ def loads(s, *, encoding=None, cls=None, object_hook=None, parse_float=None, s, 0) else: if not isinstance(s, (bytes, bytearray)): - raise TypeError('the JSON object must be str, bytes or bytearray, ' - 'not {!r}'.format(s.__class__.__name__)) + raise TypeError(f'the JSON object must be str, bytes or bytearray, ' + f'not {s.__class__.__name__}') s = s.decode(detect_encoding(s), 'surrogatepass') if (cls is None and object_hook is None and diff --git a/Lib/json/encoder.py b/Lib/json/encoder.py index 41a497c5da0160..fb083ed61bb1f8 100644 --- a/Lib/json/encoder.py +++ b/Lib/json/encoder.py @@ -176,8 +176,8 @@ def default(self, o): return JSONEncoder.default(self, o) """ - raise TypeError("Object of type '%s' is not JSON serializable" % - o.__class__.__name__) + raise TypeError(f'Object of type {o.__class__.__name__} ' + f'is not JSON serializable') def encode(self, o): """Return a JSON string representation of a Python data structure. @@ -373,7 +373,8 @@ def _iterencode_dict(dct, _current_indent_level): elif _skipkeys: continue else: - raise TypeError("key " + repr(key) + " is not a string") + raise TypeError(f'keys must be str, int, float, bool or None, ' + f'not {key.__class__.__name__}') if first: first = False else: diff --git a/Lib/test/test_json/test_fail.py b/Lib/test/test_json/test_fail.py index 791052102140b7..eb9064edea9115 100644 --- a/Lib/test/test_json/test_fail.py +++ b/Lib/test/test_json/test_fail.py @@ -93,12 +93,15 @@ def test_failures(self): def test_non_string_keys_dict(self): data = {'a' : 1, (1, 2) : 2} + with self.assertRaisesRegex(TypeError, + 'keys must be str, int, float, bool or None, not tuple'): + self.dumps(data) - #This is for c encoder - self.assertRaises(TypeError, self.dumps, data) - - #This is for python encoder - self.assertRaises(TypeError, self.dumps, data, indent=True) + def test_not_serializable(self): + import sys + with self.assertRaisesRegex(TypeError, + 'Object of type module is not JSON serializable'): + self.dumps(sys) def test_truncated_input(self): test_cases = [ diff --git a/Modules/_json.c b/Modules/_json.c index 13218a6ecce587..5a9464e34fb7f3 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -1650,8 +1650,9 @@ encoder_listencode_dict(PyEncoderObject *s, _PyAccu *acc, continue; } else { - /* TODO: include repr of key */ - PyErr_SetString(PyExc_TypeError, "keys must be a string"); + PyErr_Format(PyExc_TypeError, + "keys must be str, int, float, bool or None, " + "not %.100s", key->ob_type->tp_name); goto bail; } From 77f5139954a878b856b0ac4c76486b27b6f4ec26 Mon Sep 17 00:00:00 2001 From: xdegaye Date: Sat, 25 Nov 2017 17:25:30 +0100 Subject: [PATCH 46/75] bpo-32059: setup.py now also searches the sysroot paths (GH-4452) detect_modules() in setup.py now also searches the sysroot paths when cross-compiling. --- .../2017-11-18-11-19-28.bpo-32059.a0Hxgp.rst | 2 + setup.py | 44 ++++++++++++++++--- 2 files changed, 39 insertions(+), 7 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2017-11-18-11-19-28.bpo-32059.a0Hxgp.rst diff --git a/Misc/NEWS.d/next/Build/2017-11-18-11-19-28.bpo-32059.a0Hxgp.rst b/Misc/NEWS.d/next/Build/2017-11-18-11-19-28.bpo-32059.a0Hxgp.rst new file mode 100644 index 00000000000000..a071bd8b325048 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2017-11-18-11-19-28.bpo-32059.a0Hxgp.rst @@ -0,0 +1,2 @@ +``detect_modules()`` in ``setup.py`` now also searches the sysroot paths +when cross-compiling. diff --git a/setup.py b/setup.py index cbe6139ddec69f..0678ddec4149ec 100644 --- a/setup.py +++ b/setup.py @@ -60,6 +60,31 @@ def add_dir_to_list(dirlist, dir): return dirlist.insert(0, dir) +def sysroot_paths(make_vars, subdirs): + """Get the paths of sysroot sub-directories. + + * make_vars: a sequence of names of variables of the Makefile where + sysroot may be set. + * subdirs: a sequence of names of subdirectories used as the location for + headers or libraries. + """ + + dirs = [] + for var_name in make_vars: + var = sysconfig.get_config_var(var_name) + if var is not None: + m = re.search(r'--sysroot=([^"]\S*|"[^"]+")', var) + if m is not None: + sysroot = m.group(1).strip('"') + for subdir in subdirs: + if os.path.isabs(subdir): + subdir = subdir[1:] + path = os.path.join(sysroot, subdir) + if os.path.isdir(path): + dirs.append(path) + break + return dirs + def macosx_sdk_root(): """ Return the directory of the current OSX SDK, @@ -559,18 +584,23 @@ def detect_modules(self): add_dir_to_list(self.compiler.include_dirs, sysconfig.get_config_var("INCLUDEDIR")) + system_lib_dirs = ['/lib64', '/usr/lib64', '/lib', '/usr/lib'] + system_include_dirs = ['/usr/include'] # lib_dirs and inc_dirs are used to search for files; # if a file is found in one of those directories, it can # be assumed that no additional -I,-L directives are needed. if not cross_compiling: - lib_dirs = self.compiler.library_dirs + [ - '/lib64', '/usr/lib64', - '/lib', '/usr/lib', - ] - inc_dirs = self.compiler.include_dirs + ['/usr/include'] + lib_dirs = self.compiler.library_dirs + system_lib_dirs + inc_dirs = self.compiler.include_dirs + system_include_dirs else: - lib_dirs = self.compiler.library_dirs[:] - inc_dirs = self.compiler.include_dirs[:] + # Add the sysroot paths. 'sysroot' is a compiler option used to + # set the logical path of the standard system headers and + # libraries. + lib_dirs = (self.compiler.library_dirs + + sysroot_paths(('LDFLAGS', 'CC'), system_lib_dirs)) + inc_dirs = (self.compiler.include_dirs + + sysroot_paths(('CPPFLAGS', 'CFLAGS', 'CC'), + system_include_dirs)) exts = [] missing = [] From 76fdac4c9f53eb8433a54bd3daf9f5cc2e702a44 Mon Sep 17 00:00:00 2001 From: xdegaye Date: Sat, 25 Nov 2017 17:32:27 +0100 Subject: [PATCH 47/75] bpo-26856: Skip test_pwd on Android until issue 32033 is fixed (GH-4561) --- Lib/test/test_pwd.py | 27 +++++++-------------------- 1 file changed, 7 insertions(+), 20 deletions(-) diff --git a/Lib/test/test_pwd.py b/Lib/test/test_pwd.py index ac9cff789eea28..c13a7c9294f983 100644 --- a/Lib/test/test_pwd.py +++ b/Lib/test/test_pwd.py @@ -4,19 +4,11 @@ pwd = support.import_module('pwd') -def _getpwall(): - # Android does not have getpwall. - if hasattr(pwd, 'getpwall'): - return pwd.getpwall() - elif hasattr(pwd, 'getpwuid'): - return [pwd.getpwuid(0)] - else: - return [] - +@unittest.skipUnless(hasattr(pwd, 'getpwall'), 'Does not have getpwall()') class PwdTest(unittest.TestCase): def test_values(self): - entries = _getpwall() + entries = pwd.getpwall() for e in entries: self.assertEqual(len(e), 7) @@ -42,7 +34,7 @@ def test_values(self): # and check afterwards (done in test_values_extended) def test_values_extended(self): - entries = _getpwall() + entries = pwd.getpwall() entriesbyname = {} entriesbyuid = {} @@ -66,13 +58,12 @@ def test_errors(self): self.assertRaises(TypeError, pwd.getpwuid, 3.14) self.assertRaises(TypeError, pwd.getpwnam) self.assertRaises(TypeError, pwd.getpwnam, 42) - if hasattr(pwd, 'getpwall'): - self.assertRaises(TypeError, pwd.getpwall, 42) + self.assertRaises(TypeError, pwd.getpwall, 42) # try to get some errors bynames = {} byuids = {} - for (n, p, u, g, gecos, d, s) in _getpwall(): + for (n, p, u, g, gecos, d, s) in pwd.getpwall(): bynames[n] = u byuids[u] = n @@ -106,17 +97,13 @@ def test_errors(self): # loop, say), pwd.getpwuid() might still be able to find data for that # uid. Using sys.maxint may provoke the same problems, but hopefully # it will be a more repeatable failure. - # Android accepts a very large span of uids including sys.maxsize and - # -1; it raises KeyError with 1 or 2 for example. fakeuid = sys.maxsize self.assertNotIn(fakeuid, byuids) - if not support.is_android: - self.assertRaises(KeyError, pwd.getpwuid, fakeuid) + self.assertRaises(KeyError, pwd.getpwuid, fakeuid) # -1 shouldn't be a valid uid because it has a special meaning in many # uid-related functions - if not support.is_android: - self.assertRaises(KeyError, pwd.getpwuid, -1) + self.assertRaises(KeyError, pwd.getpwuid, -1) # should be out of uid_t range self.assertRaises(KeyError, pwd.getpwuid, 2**128) self.assertRaises(KeyError, pwd.getpwuid, -2**128) From cef88b9c15cf387cf6a39a387a6868883409df4f Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Sat, 25 Nov 2017 13:02:55 -0800 Subject: [PATCH 48/75] mark fatal_error as noreturn (#4563) clang can't figure out that fatal_error is noreturn itself and emits warnings: ../cpython/Python/pylifecycle.c:2116:1: warning: function declared 'noreturn' should not return [-Winvalid-noreturn] } ^ ../cpython/Python/pylifecycle.c:2125:1: warning: function declared 'noreturn' should not return [-Winvalid-noreturn] } ^ --- Python/pylifecycle.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 868ac8450d453c..b89cbc88d4b90e 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -2048,7 +2048,7 @@ fatal_output_debug(const char *msg) } #endif -static void +static void _Py_NO_RETURN fatal_error(const char *prefix, const char *msg, int status) { const int fd = fileno(stderr); From 53efbf3977a44e382397e7994a2524b4f8c9d053 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Sun, 26 Nov 2017 13:04:46 +1000 Subject: [PATCH 49/75] bpo-11063: Handle uuid.h being in default include path (GH-4565) find_file() returns an empty list if it finds the requested header on the standard include path, so header existence checks need to be explicitly against "is not None". --- setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 0678ddec4149ec..c22de17f953396 100644 --- a/setup.py +++ b/setup.py @@ -1680,12 +1680,11 @@ class db_found(Exception): pass # Build the _uuid module if possible uuid_incs = find_file("uuid.h", inc_dirs, ["/usr/include/uuid"]) - if uuid_incs: + if uuid_incs is not None: if self.compiler.find_library_file(lib_dirs, 'uuid'): uuid_libs = ['uuid'] else: uuid_libs = [] - if uuid_incs: self.extensions.append(Extension('_uuid', ['_uuidmodule.c'], libraries=uuid_libs, include_dirs=uuid_incs)) From 4274609e1856facd80b7ee588b0791fe8963b9e0 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Sun, 26 Nov 2017 14:19:13 +1000 Subject: [PATCH 50/75] bpo-32096: Ensure new embedding test can find the encodings module (GH-4566) --- Programs/_testembed.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Programs/_testembed.c b/Programs/_testembed.c index 52a0b5124a37f0..21aa76e9de38bf 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -132,7 +132,8 @@ static int test_forced_io_encoding(void) static int test_pre_initialization_api(void) { - wchar_t *program = Py_DecodeLocale("spam", NULL); + /* Leading "./" ensures getpath.c can still find the standard library */ + wchar_t *program = Py_DecodeLocale("./spam", NULL); if (program == NULL) { fprintf(stderr, "Fatal error: cannot decode program name\n"); return 1; From a6fba9b827e395fc9583c07bc2d15cd11f684439 Mon Sep 17 00:00:00 2001 From: xdegaye Date: Sun, 26 Nov 2017 10:31:44 +0100 Subject: [PATCH 51/75] bpo-32126: Skip asyncio test when sem_open() is not functional (GH-4559) --- Lib/test/test_asyncio/test_events.py | 4 ++++ .../next/Tests/2017-11-24-18-15-12.bpo-32126.PLmNLn.rst | 2 ++ 2 files changed, 6 insertions(+) create mode 100644 Misc/NEWS.d/next/Tests/2017-11-24-18-15-12.bpo-32126.PLmNLn.rst diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py index a1079128d0d726..a6e4ecf7958df7 100644 --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -2155,6 +2155,10 @@ def tearDown(self): super().tearDown() def test_get_event_loop_new_process(self): + # Issue bpo-32126: The multiprocessing module used by + # ProcessPoolExecutor is not functional when the + # multiprocessing.synchronize module cannot be imported. + support.import_module('multiprocessing.synchronize') async def main(): pool = concurrent.futures.ProcessPoolExecutor() result = await self.loop.run_in_executor( diff --git a/Misc/NEWS.d/next/Tests/2017-11-24-18-15-12.bpo-32126.PLmNLn.rst b/Misc/NEWS.d/next/Tests/2017-11-24-18-15-12.bpo-32126.PLmNLn.rst new file mode 100644 index 00000000000000..b5ba9d574e35b4 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2017-11-24-18-15-12.bpo-32126.PLmNLn.rst @@ -0,0 +1,2 @@ +Skip test_get_event_loop_new_process in test.test_asyncio.test_events when +sem_open() is not functional. From 99e86b2bd1c17cc3a4860a4d07abd39cd7c5df37 Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Sun, 26 Nov 2017 11:44:25 -0500 Subject: [PATCH 52/75] Two tests cannot succeed on Travis-CI --- Lib/test/test_uuid.py | 4 ++++ Lib/uuid.py | 2 -- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index 97f34cdbe3310f..05427968692d26 100644 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -524,11 +524,15 @@ def check_node(self, node, requires=None, network=False): "%s is not an RFC 4122 node ID" % hex) @unittest.skipUnless(os.name == 'posix', 'requires Posix') + @unittest.skipIf(os.getenv('TRAVIS'), + 'Travis-CI has no universally administered MAC addresses') def test_ifconfig_getnode(self): node = self.uuid._ifconfig_getnode() self.check_node(node, 'ifconfig', True) @unittest.skipUnless(os.name == 'posix', 'requires Posix') + @unittest.skipIf(os.getenv('TRAVIS'), + 'Travis-CI has no universally administered MAC addresses') def test_ip_getnode(self): node = self.uuid._ip_getnode() self.check_node(node, 'ip', True) diff --git a/Lib/uuid.py b/Lib/uuid.py index c551bb60a0c9bd..9e7c672f7a0a6f 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -374,9 +374,7 @@ def _find_mac(command, args, hw_identifiers, get_index): word = words[get_index(i)] mac = int(word.replace(b':', b''), 16) if _is_universal(mac): - print('%012x' % mac, 'universal', command) return mac - print('%012x' % mac, 'local', command) first_local_mac = first_local_mac or mac except (ValueError, IndexError): # Virtual interfaces, such as those provided by From 74702e3b6b12177cbabd49d6484a07c23ac379ce Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 26 Nov 2017 22:42:57 +0200 Subject: [PATCH 53/75] bpo-32107: Remove the check of the multicast bit in test_uuid. While it is required that the multicast bit (1<<40) should be set in random generated addresses, there is no requirement that it should be cleared in addresses obtained from network cards. --- Lib/test/test_uuid.py | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index 083c2aa8aab54d..fa89d62aec2ab6 100644 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -512,58 +512,55 @@ def test_find_mac(self): self.assertEqual(mac, 0x1234567890ab) - def check_node(self, node, requires=None, network=False): + def check_node(self, node, requires=None): if requires and node is None: self.skipTest('requires ' + requires) hex = '%012x' % node if support.verbose >= 2: print(hex, end=' ') - if network: - # 47 bit will never be set in IEEE 802 addresses obtained - # from network cards. - self.assertFalse(node & 0x010000000000, hex) self.assertTrue(0 < node < (1 << 48), "%s is not an RFC 4122 node ID" % hex) @unittest.skipUnless(os.name == 'posix', 'requires Posix') def test_ifconfig_getnode(self): node = self.uuid._ifconfig_getnode() - self.check_node(node, 'ifconfig', True) + self.check_node(node, 'ifconfig') @unittest.skipUnless(os.name == 'posix', 'requires Posix') def test_ip_getnode(self): node = self.uuid._ip_getnode() - self.check_node(node, 'ip', True) + self.check_node(node, 'ip') @unittest.skipUnless(os.name == 'posix', 'requires Posix') def test_arp_getnode(self): node = self.uuid._arp_getnode() - self.check_node(node, 'arp', True) + self.check_node(node, 'arp') @unittest.skipUnless(os.name == 'posix', 'requires Posix') def test_lanscan_getnode(self): node = self.uuid._lanscan_getnode() - self.check_node(node, 'lanscan', True) + self.check_node(node, 'lanscan') @unittest.skipUnless(os.name == 'posix', 'requires Posix') def test_netstat_getnode(self): node = self.uuid._netstat_getnode() - self.check_node(node, 'netstat', True) + self.check_node(node, 'netstat') @unittest.skipUnless(os.name == 'nt', 'requires Windows') def test_ipconfig_getnode(self): node = self.uuid._ipconfig_getnode() - self.check_node(node, 'ipconfig', True) + self.check_node(node, 'ipconfig') @unittest.skipUnless(importable('win32wnet'), 'requires win32wnet') @unittest.skipUnless(importable('netbios'), 'requires netbios') def test_netbios_getnode(self): node = self.uuid._netbios_getnode() - self.check_node(node, network=True) + self.check_node(node) def test_random_getnode(self): node = self.uuid._random_getnode() # Least significant bit of first octet must be set. + # See RFC 4122, $4.1.6. self.assertTrue(node & 0x010000000000, '%012x' % node) self.check_node(node) From d8d6b9122134f040cd5a4f15f40f6c9e3386db4d Mon Sep 17 00:00:00 2001 From: Caleb Hattingh Date: Mon, 27 Nov 2017 07:18:30 +1000 Subject: [PATCH 54/75] bpo-30487: automatically create a venv and install Sphinx when running make (GH-4346) --- .travis.yml | 2 +- Doc/Makefile | 12 +++++++----- Doc/README.rst | 39 +++++++++++++++++++++++---------------- 3 files changed, 31 insertions(+), 22 deletions(-) diff --git a/.travis.yml b/.travis.yml index c207bd72da227c..4e8be5e0350232 100644 --- a/.travis.yml +++ b/.travis.yml @@ -36,7 +36,7 @@ matrix: # (Updating the version is fine as long as no warnings are raised by doing so.) - python -m pip install sphinx~=1.6.1 blurb script: - - make check suspicious html SPHINXOPTS="-q -W -j4" + - make check suspicious html SPHINXBUILD="sphinx-build" SPHINXOPTS="-q -W -j4" - os: linux language: c compiler: gcc diff --git a/Doc/Makefile b/Doc/Makefile index 307d1e0e7de10b..69e7e2eecea204 100644 --- a/Doc/Makefile +++ b/Doc/Makefile @@ -17,7 +17,7 @@ ALLSPHINXOPTS = -b $(BUILDER) -d build/doctrees -D latex_elements.papersize=$(PA .PHONY: help build html htmlhelp latex text changes linkcheck \ suspicious coverage doctest pydoc-topics htmlview clean dist check serve \ - autobuild-dev autobuild-stable venv + autobuild-dev autobuild-stable help: @echo "Please use \`make ' where is one of" @@ -39,7 +39,7 @@ help: @echo " check to run a check for frequent markup errors" @echo " serve to serve the documentation on the localhost (8000)" -build: +build: venv -mkdir -p build # Look first for a Misc/NEWS file (building from a source release tarball # or old repo) and use that, otherwise look for a Misc/NEWS.d directory @@ -122,9 +122,11 @@ clean: -rm -rf build/* $(VENVDIR)/* venv: - $(PYTHON) -m venv $(VENVDIR) - $(VENVDIR)/bin/python3 -m pip install -U Sphinx blurb - @echo "The venv has been created in the $(VENVDIR) directory" + @if [ "$(SPHINXBUILD)" == "PATH=$(VENVDIR)/bin:$$PATH sphinx-build" ]; then \ + $(PYTHON) -m venv $(VENVDIR); \ + echo "A virtual environment for Docs has been made in the $(VENVDIR) directory"; \ + $(VENVDIR)/bin/python3 -m pip install Sphinx blurb; \ + fi dist: rm -rf dist diff --git a/Doc/README.rst b/Doc/README.rst index a29d1f3a708a43..c0a8d89a8c7925 100644 --- a/Doc/README.rst +++ b/Doc/README.rst @@ -21,21 +21,16 @@ tree but are maintained separately and are available from * `Sphinx `_ * `blurb `_ -The easiest way to install these tools is to create a virtual environment and -install the tools into there. +You could manually create a virtual environment and install them, but there is +a ``Makefile`` already set up to do this for you, as long as you have a working +Python 3 interpreter available. Using make ---------- -To get started on UNIX, you can create a virtual environment with the command :: - - make venv - -That will install all the tools necessary to build the documentation. Assuming -the virtual environment was created in the ``env`` directory (the default; -configurable with the VENVDIR variable), you can run the following command to -build the HTML output files:: +A Makefile has been prepared so that (on Unix), after you change into the +``Doc/`` directory you can simply run :: make html @@ -44,8 +39,17 @@ look for instances of sphinxbuild and blurb installed on your process PATH (configurable with the SPHINXBUILD and BLURB variables). On Windows, we try to emulate the Makefile as closely as possible with a -``make.bat`` file. If you need to specify the Python interpreter to use, -set the PYTHON environment variable instead. +``make.bat`` file. + +To use a Python interpreter that's not called ``python3``, use the standard +way to set Makefile variables, using e.g. :: + + make html PYTHON=python + +On Windows, set the PYTHON environment variable instead. + +To use a specific sphinx-build (something other than ``sphinx-build``), set +the SPHINXBUILD variable. Available make targets are: @@ -104,11 +108,14 @@ Available make targets are: Without make ------------ -First, install the tool dependencies from PyPI. - -Then, from the ``Doc`` directory, run :: +Install the Sphinx package and its dependencies from PyPI. In this situation, +you'll have to create a virtual environment manually, and install Sphinx into +it. Change into the ``Doc`` directory and run :: - sphinx-build -b . build/ + $ python3 -m venv venv + $ source venv/bin/activate + (venv) $ pip install Sphinx + (venv) $ sphinx-build -b . build/ where ```` is one of html, text, latex, or htmlhelp (for explanations see the make targets above). From 0cd2e81bea639828d7c9a7afc61fb1da9699492c Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 26 Nov 2017 23:23:02 +0100 Subject: [PATCH 55/75] bpo-29879: Update typing documentation. (GH-4573) - Add "version added: 3.5.2" note where it was missing. - Remove the mention that Reversible is new in 3.5.2 --- Doc/library/typing.rst | 12 ++++++++++++ Misc/NEWS.d/3.5.2rc1.rst | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 9883d8bbe86596..d28a5d548431ef 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -146,6 +146,8 @@ See :pep:`484` for more details. ``Derived`` is expected. This is useful when you want to prevent logic errors with minimal runtime cost. +.. versionadded:: 3.5.2 + Callable -------- @@ -494,6 +496,8 @@ The module defines the following classes, functions and decorators: ``Type[Any]`` is equivalent to ``Type`` which in turn is equivalent to ``type``, which is the root of Python's metaclass hierarchy. + .. versionadded:: 3.5.2 + .. class:: Iterable(Generic[T_co]) A generic version of :class:`collections.abc.Iterable`. @@ -674,6 +678,8 @@ The module defines the following classes, functions and decorators: A generic version of :class:`collections.defaultdict`. + .. versionadded:: 3.5.2 + .. class:: Counter(collections.Counter, Dict[T, int]) A generic version of :class:`collections.Counter`. @@ -762,6 +768,8 @@ The module defines the following classes, functions and decorators: def add_unicode_checkmark(text: Text) -> Text: return text + u' \u2713' + .. versionadded:: 3.5.2 + .. class:: io Wrapper namespace for I/O stream types. @@ -847,6 +855,8 @@ The module defines the following classes, functions and decorators: UserId = NewType('UserId', int) first_user = UserId(1) + .. versionadded:: 3.5.2 + .. function:: cast(typ, val) Cast a value to a type. @@ -1054,3 +1064,5 @@ The module defines the following classes, functions and decorators: "forward reference", to hide the ``expensive_mod`` reference from the interpreter runtime. Type annotations for local variables are not evaluated, so the second annotation does not need to be enclosed in quotes. + + .. versionadded:: 3.5.2 diff --git a/Misc/NEWS.d/3.5.2rc1.rst b/Misc/NEWS.d/3.5.2rc1.rst index c33dcb2309f636..caed84a06f78d5 100644 --- a/Misc/NEWS.d/3.5.2rc1.rst +++ b/Misc/NEWS.d/3.5.2rc1.rst @@ -420,7 +420,7 @@ patch by ingrid. .. section: Library A new version of typing.py provides several new classes and features: -@overload outside stubs, Reversible, DefaultDict, Text, ContextManager, +@overload outside stubs, DefaultDict, Text, ContextManager, Type[], NewType(), TYPE_CHECKING, and numerous bug fixes (note that some of the new features are not yet implemented in mypy or other static analyzers). Also classes for PEP 492 (Awaitable, AsyncIterable, AsyncIterator) have been From ede2ac913eba47131ee1bbc37a9aea344d678576 Mon Sep 17 00:00:00 2001 From: Mandeep Singh Date: Mon, 27 Nov 2017 04:01:27 +0530 Subject: [PATCH 56/75] bpo-23033: Improve SSL Certificate handling (GH-937) Wildcard is now supported in hostname when it is one and only character in the leftmost segment. --- Doc/library/ssl.rst | 4 ++++ Lib/ssl.py | 9 +++++++-- Lib/test/test_ssl.py | 13 +++++++------ Misc/ACKS | 1 + .../2017-11-26-17-00-52.bpo-23033.YGXRWT.rst | 3 +++ 5 files changed, 22 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2017-11-26-17-00-52.bpo-23033.YGXRWT.rst diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index 9b3bdd5489d4a9..45bb65ff0715e3 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -429,6 +429,10 @@ Certificate handling Matching of IP addresses, when present in the subjectAltName field of the certificate, is now supported. + .. versionchanged:: 3.7 + Allow wildcard when it is the leftmost and the only character + in that segment. + .. function:: cert_time_to_seconds(cert_time) Return the time in seconds since the Epoch, given the ``cert_time`` diff --git a/Lib/ssl.py b/Lib/ssl.py index 75caae0c440566..fa83606e7cd5a5 100644 --- a/Lib/ssl.py +++ b/Lib/ssl.py @@ -221,7 +221,7 @@ class CertificateError(ValueError): pass -def _dnsname_match(dn, hostname, max_wildcards=1): +def _dnsname_match(dn, hostname): """Matching according to RFC 6125, section 6.4.3 http://tools.ietf.org/html/rfc6125#section-6.4.3 @@ -233,7 +233,12 @@ def _dnsname_match(dn, hostname, max_wildcards=1): leftmost, *remainder = dn.split(r'.') wildcards = leftmost.count('*') - if wildcards > max_wildcards: + if wildcards == 1 and len(leftmost) > 1: + # Only match wildcard in leftmost segment. + raise CertificateError( + "wildcard can only be present in the leftmost segment: " + repr(dn)) + + if wildcards > 1: # Issue #17980: avoid denials of service by refusing more # than one wildcard per fragment. A survey of established # policy among SSL implementations showed it to be a diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index aa2429ac9827ae..c65290b945f6c9 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -512,10 +512,11 @@ def fail(cert, hostname): fail(cert, 'Xa.com') fail(cert, '.a.com') - # only match one left-most wildcard + # only match wildcards when they are the only thing + # in left-most segment cert = {'subject': ((('commonName', 'f*.com'),),)} - ok(cert, 'foo.com') - ok(cert, 'f.com') + fail(cert, 'foo.com') + fail(cert, 'f.com') fail(cert, 'bar.com') fail(cert, 'foo.a.com') fail(cert, 'bar.foo.com') @@ -552,8 +553,8 @@ def fail(cert, hostname): # are supported. idna = 'www*.pythön.org'.encode("idna").decode("ascii") cert = {'subject': ((('commonName', idna),),)} - ok(cert, 'www.pythön.org'.encode("idna").decode("ascii")) - ok(cert, 'www1.pythön.org'.encode("idna").decode("ascii")) + fail(cert, 'www.pythön.org'.encode("idna").decode("ascii")) + fail(cert, 'www1.pythön.org'.encode("idna").decode("ascii")) fail(cert, 'ftp.pythön.org'.encode("idna").decode("ascii")) fail(cert, 'pythön.org'.encode("idna").decode("ascii")) @@ -637,7 +638,7 @@ def fail(cert, hostname): # Issue #17980: avoid denials of service by refusing more than one # wildcard per fragment. cert = {'subject': ((('commonName', 'a*b.com'),),)} - ok(cert, 'axxb.com') + fail(cert, 'axxb.com') cert = {'subject': ((('commonName', 'a*b.co*'),),)} fail(cert, 'axxb.com') cert = {'subject': ((('commonName', 'a*b*.com'),),)} diff --git a/Misc/ACKS b/Misc/ACKS index fc7154f762ecb4..54d8d62b633f70 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1467,6 +1467,7 @@ Nathan Paul Simons Guilherme Simões Adam Simpkins Ravi Sinha +Mandeep Singh Janne Sinkkonen Ng Pheng Siong Yann Sionneau diff --git a/Misc/NEWS.d/next/Library/2017-11-26-17-00-52.bpo-23033.YGXRWT.rst b/Misc/NEWS.d/next/Library/2017-11-26-17-00-52.bpo-23033.YGXRWT.rst new file mode 100644 index 00000000000000..cecc10aebb9912 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-11-26-17-00-52.bpo-23033.YGXRWT.rst @@ -0,0 +1,3 @@ +Wildcard is now supported in hostname when it is one and only character in +the left most segment of hostname in second argument of +:meth:`ssl.match_hostname`. Patch by Mandeep Singh. From 41adbad8ee5b1d6c37892ae54062e1d53fc365ab Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Tue, 21 Nov 2017 12:00:21 -0500 Subject: [PATCH 57/75] bpo-32107 - Fix test_uuid The original check was erroneous in two ways. First, the documentation said that "47 bit will never be set in IEEE 802 addresses obtained from network cards". I think this is referring to the universally vs. locally administered MAC addresses. Network cards from hardware manufacturers will always be universally administered in order to guarantee global uniqueness. But from https://en.wikipedia.org/wiki/MAC_address it says that this flag is the "distinguished by setting the second-least-significant bit of the first octet of the address", where a 0 means universally administered and a 1 means it's locally administered. This works out to the 42nd bit when counting the LSB as bit 1, or 1<<41. The second bug is that the original bitmask value isn't right for either description: % ./python.exe -c "from math import log2; print(log2(0x010000000000))" 40.0 This causes the test to fail on valid MAC addresses. Fix this by improving the comment, with references, and using the correct bitmask. --- Lib/test/test_uuid.py | 13 ++++++++++--- Lib/uuid.py | 3 +-- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index 083c2aa8aab54d..5adba57e07d9f2 100644 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -519,9 +519,16 @@ def check_node(self, node, requires=None, network=False): if support.verbose >= 2: print(hex, end=' ') if network: - # 47 bit will never be set in IEEE 802 addresses obtained - # from network cards. - self.assertFalse(node & 0x010000000000, hex) + # The second least significant bit of the first octet signifies + # whether the MAC (IEEE 802, or EUI-48) address is universally (0) + # or locally (1) administered. Network cards from hardware + # manufacturers will always be universally administered to + # guarantee global uniqueness of the MAC address. This bit works + # out to be the 42nd bit counting from 1 being the least + # significant, or 1<<41. For a good, simple explanation, see the + # section on Universal vs. local in this page: + # https://en.wikipedia.org/wiki/MAC_address + self.assertFalse(node & (1 << 41), hex) self.assertTrue(0 < node < (1 << 48), "%s is not an RFC 4122 node ID" % hex) diff --git a/Lib/uuid.py b/Lib/uuid.py index 020c6e73c863d4..fee809dd1abfab 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -404,8 +404,7 @@ def _arp_getnode(): # This works on Linux, FreeBSD and NetBSD mac = _find_mac('arp', '-an', [os.fsencode('(%s)' % ip_addr)], lambda i: i+2) - if mac: - return mac + return mac def _lanscan_getnode(): """Get the hardware address on Unix by running lanscan.""" From d3d0465e4a64f24d5727553f1031302b29d3c849 Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Tue, 21 Nov 2017 14:10:40 -0500 Subject: [PATCH 58/75] Return None instead of 0 --- Lib/uuid.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/uuid.py b/Lib/uuid.py index fee809dd1abfab..27335f89192909 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -404,7 +404,8 @@ def _arp_getnode(): # This works on Linux, FreeBSD and NetBSD mac = _find_mac('arp', '-an', [os.fsencode('(%s)' % ip_addr)], lambda i: i+2) - return mac + # Return None instead of 0. + return mac if mac else None def _lanscan_getnode(): """Get the hardware address on Unix by running lanscan.""" From 7c17cd814f3a74c08566bb9386b4c03bce0a4eb2 Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Tue, 21 Nov 2017 23:08:06 -0500 Subject: [PATCH 59/75] Never return a locally administered MAC address Also, clean up some return sites. --- Lib/test/test_uuid.py | 9 --------- Lib/uuid.py | 45 ++++++++++++++++++++++++++++++++++--------- 2 files changed, 36 insertions(+), 18 deletions(-) diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index 5adba57e07d9f2..19564f2d548d28 100644 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -519,15 +519,6 @@ def check_node(self, node, requires=None, network=False): if support.verbose >= 2: print(hex, end=' ') if network: - # The second least significant bit of the first octet signifies - # whether the MAC (IEEE 802, or EUI-48) address is universally (0) - # or locally (1) administered. Network cards from hardware - # manufacturers will always be universally administered to - # guarantee global uniqueness of the MAC address. This bit works - # out to be the 42nd bit counting from 1 being the least - # significant, or 1<<41. For a good, simple explanation, see the - # section on Universal vs. local in this page: - # https://en.wikipedia.org/wiki/MAC_address self.assertFalse(node & (1 << 41), hex) self.assertTrue(0 < node < (1 << 48), "%s is not an RFC 4122 node ID" % hex) diff --git a/Lib/uuid.py b/Lib/uuid.py index 27335f89192909..0535433f2bffda 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -342,11 +342,26 @@ def _popen(command, *args): env=env) return proc +# For MAC (a.k.a. IEEE 802, or EUI-48) addresses, the second least significant +# bit of the first octet signifies whether the MAC address is universally (0) +# or locally (1) administered. Network cards from hardware manufacturers will +# always be universally administered to guarantee global uniqueness of the MAC +# address, but any particular machine may have other interfaces which are +# locally administered. An example of the latter is the bridge interface to +# the Touch Bar on MacBook Pros. +# +# This bit works out to be the 42nd bit counting from 1 being the least +# significant, or 1<<41. We'll skip over any locally administered MAC +# addresses, as it makes no sense to use those in UUID calculation. +# +# For a good, simple explanation, see the section on Universal vs. local in +# this page: https://en.wikipedia.org/wiki/MAC_address + def _find_mac(command, args, hw_identifiers, get_index): try: proc = _popen(command, *args.split()) if not proc: - return + return None with proc: for line in proc.stdout: words = line.lower().rstrip().split() @@ -355,7 +370,7 @@ def _find_mac(command, args, hw_identifiers, get_index): try: word = words[get_index(i)] mac = int(word.replace(b':', b''), 16) - if mac: + if ~(mac & (1<<41)): return mac except (ValueError, IndexError): # Virtual interfaces, such as those provided by @@ -366,6 +381,7 @@ def _find_mac(command, args, hw_identifiers, get_index): pass except OSError: pass + return None def _ifconfig_getnode(): """Get the hardware address on Unix by running ifconfig.""" @@ -375,6 +391,7 @@ def _ifconfig_getnode(): mac = _find_mac('ifconfig', args, keywords, lambda i: i+1) if mac: return mac + return None def _ip_getnode(): """Get the hardware address on Unix by running ip.""" @@ -382,6 +399,7 @@ def _ip_getnode(): mac = _find_mac('ip', 'link list', [b'link/ether'], lambda i: i+1) if mac: return mac + return None def _arp_getnode(): """Get the hardware address on Unix by running arp.""" @@ -405,7 +423,9 @@ def _arp_getnode(): mac = _find_mac('arp', '-an', [os.fsencode('(%s)' % ip_addr)], lambda i: i+2) # Return None instead of 0. - return mac if mac else None + if mac: + return mac + return None def _lanscan_getnode(): """Get the hardware address on Unix by running lanscan.""" @@ -418,25 +438,26 @@ def _netstat_getnode(): try: proc = _popen('netstat', '-ia') if not proc: - return + return None with proc: words = proc.stdout.readline().rstrip().split() try: i = words.index(b'Address') except ValueError: - return + return None for line in proc.stdout: try: words = line.rstrip().split() word = words[i] if len(word) == 17 and word.count(b':') == 5: mac = int(word.replace(b':', b''), 16) - if mac: + if ~(mac & (1<<41)): return mac except (ValueError, IndexError): pass except OSError: pass + return None def _ipconfig_getnode(): """Get the hardware address on Windows by running ipconfig.exe.""" @@ -458,7 +479,10 @@ def _ipconfig_getnode(): for line in pipe: value = line.split(':')[-1].strip().lower() if re.match('([0-9a-f][0-9a-f]-){5}[0-9a-f][0-9a-f]', value): - return int(value.replace('-', ''), 16) + mac = int(value.replace('-', ''), 16) + if ~(mac & (1<<41)): + return mac + return None def _netbios_getnode(): """Get the hardware address on Windows using NetBIOS calls. @@ -469,7 +493,7 @@ def _netbios_getnode(): ncb.Buffer = adapters = netbios.LANA_ENUM() adapters._pack() if win32wnet.Netbios(ncb) != 0: - return + return None adapters._unpack() for i in range(adapters.length): ncb.Reset() @@ -488,7 +512,10 @@ def _netbios_getnode(): bytes = status.adapter_address[:6] if len(bytes) != 6: continue - return int.from_bytes(bytes, 'big') + mac = int.from_bytes(bytes, 'big') + if ~(mac & (1<<41)): + return mac + return None _generate_time_safe = _UuidCreate = None From 511fab0a6f980665e01f0cade38c7ecffb37257c Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Thu, 23 Nov 2017 10:16:21 -0500 Subject: [PATCH 60/75] Several sleep-improved fixes * Add a helper function to determine whether a MAC address is universally administered or not. * In the various MAC getter functions, keep track of the first locally administered MAC, and if no universally administered MAC is found, return the first local MAC. * In the ultimate fallback _random_getnode(), add a comment for the multicast bit being set, and use a better bitmask * In getnode(), just fall back to _random_getnode() explicitly if no other MAC has been found. --- Lib/test/test_uuid.py | 2 +- Lib/uuid.py | 45 +++++++++++++++++++++++++++++++------------ 2 files changed, 34 insertions(+), 13 deletions(-) diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index 19564f2d548d28..97f34cdbe3310f 100644 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -562,7 +562,7 @@ def test_netbios_getnode(self): def test_random_getnode(self): node = self.uuid._random_getnode() # Least significant bit of first octet must be set. - self.assertTrue(node & 0x010000000000, '%012x' % node) + self.assertTrue(node & (1 << 40), '%012x' % node) self.check_node(node) @unittest.skipUnless(os.name == 'posix', 'requires Posix') diff --git a/Lib/uuid.py b/Lib/uuid.py index 0535433f2bffda..3eec69f6f99d4b 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -354,10 +354,13 @@ def _popen(command, *args): # significant, or 1<<41. We'll skip over any locally administered MAC # addresses, as it makes no sense to use those in UUID calculation. # -# For a good, simple explanation, see the section on Universal vs. local in -# this page: https://en.wikipedia.org/wiki/MAC_address +# See https://en.wikipedia.org/wiki/MAC_address#Universal_vs._local + +def _is_universal(mac): + return not (mac & (1 << 41)) def _find_mac(command, args, hw_identifiers, get_index): + first_local_mac = None try: proc = _popen(command, *args.split()) if not proc: @@ -370,8 +373,9 @@ def _find_mac(command, args, hw_identifiers, get_index): try: word = words[get_index(i)] mac = int(word.replace(b':', b''), 16) - if ~(mac & (1<<41)): + if _is_universal(mac): return mac + first_local_mac = first_local_mac or mac except (ValueError, IndexError): # Virtual interfaces, such as those provided by # VPNs, do not have a colon-delimited MAC address @@ -381,7 +385,7 @@ def _find_mac(command, args, hw_identifiers, get_index): pass except OSError: pass - return None + return first_local_mac or None def _ifconfig_getnode(): """Get the hardware address on Unix by running ifconfig.""" @@ -435,6 +439,7 @@ def _lanscan_getnode(): def _netstat_getnode(): """Get the hardware address on Unix by running netstat.""" # This might work on AIX, Tru64 UNIX. + first_local_mac = None try: proc = _popen('netstat', '-ia') if not proc: @@ -451,17 +456,19 @@ def _netstat_getnode(): word = words[i] if len(word) == 17 and word.count(b':') == 5: mac = int(word.replace(b':', b''), 16) - if ~(mac & (1<<41)): + if _is_universal(mac): return mac + first_local_mac = first_local_mac or mac except (ValueError, IndexError): pass except OSError: pass - return None + return first_local_mac or None def _ipconfig_getnode(): """Get the hardware address on Windows by running ipconfig.exe.""" import os, re + first_local_mac = None dirs = ['', r'c:\windows\system32', r'c:\winnt\system32'] try: import ctypes @@ -480,14 +487,16 @@ def _ipconfig_getnode(): value = line.split(':')[-1].strip().lower() if re.match('([0-9a-f][0-9a-f]-){5}[0-9a-f][0-9a-f]', value): mac = int(value.replace('-', ''), 16) - if ~(mac & (1<<41)): + if _is_universal(mac): return mac - return None + first_local_mac = first_local_mac or mac + return first_local_mac or None def _netbios_getnode(): """Get the hardware address on Windows using NetBIOS calls. See http://support.microsoft.com/kb/118623 for details.""" import win32wnet, netbios + first_local_mac = None ncb = netbios.NCB() ncb.Command = netbios.NCBENUM ncb.Buffer = adapters = netbios.LANA_ENUM() @@ -513,8 +522,9 @@ def _netbios_getnode(): if len(bytes) != 6: continue mac = int.from_bytes(bytes, 'big') - if ~(mac & (1<<41)): + if _is_universal(mac): return mac + first_local_mac = first_local_mac or mac return None @@ -628,9 +638,19 @@ def _windll_getnode(): return UUID(bytes=bytes_(_buffer.raw)).node def _random_getnode(): - """Get a random node ID, with eighth bit set as suggested by RFC 4122.""" + """Get a random node ID.""" + # RFC 4122, $4.1.6. says "For systems with no IEEE address, a randomly or + # pseudo-randomly generated value may be used; see Section 4.5. The + # multicast bit must be set in such addresses, in order that they will + # never conflict with addresses obtained from network cards." + # + # The "multicast bit" of a MAC address is defined to be "the least + # significant bit of the first octet". This worlds out to be the 41st bit + # counting from 1 being the least significant bit, or 1<<40. + # + # See https://en.wikipedia.org/wiki/MAC_address#Unicast_vs._multicast import random - return random.getrandbits(48) | 0x010000000000 + return random.getrandbits(48) | (1 << 40) _node = None @@ -653,13 +673,14 @@ def getnode(): getters = [_unix_getnode, _ifconfig_getnode, _ip_getnode, _arp_getnode, _lanscan_getnode, _netstat_getnode] - for getter in getters + [_random_getnode]: + for getter in getters: try: _node = getter() except: continue if _node is not None: return _node + return _random_getnode() _last_timestamp = None From af7929c50060a09e6be3d6a6783318fb8697c816 Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Thu, 23 Nov 2017 10:25:51 -0500 Subject: [PATCH 61/75] A missed return --- Lib/uuid.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/uuid.py b/Lib/uuid.py index 3eec69f6f99d4b..364b5b0272b785 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -525,7 +525,7 @@ def _netbios_getnode(): if _is_universal(mac): return mac first_local_mac = first_local_mac or mac - return None + return first_local_mac or None _generate_time_safe = _UuidCreate = None From 51ba0b30bdbad94ca2c0c34d60d0527125be47d2 Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Thu, 23 Nov 2017 10:27:21 -0500 Subject: [PATCH 62/75] Two typos --- Lib/uuid.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/uuid.py b/Lib/uuid.py index 364b5b0272b785..9e7c672f7a0a6f 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -639,13 +639,13 @@ def _windll_getnode(): def _random_getnode(): """Get a random node ID.""" - # RFC 4122, $4.1.6. says "For systems with no IEEE address, a randomly or + # RFC 4122, $4.1.6 says "For systems with no IEEE address, a randomly or # pseudo-randomly generated value may be used; see Section 4.5. The # multicast bit must be set in such addresses, in order that they will # never conflict with addresses obtained from network cards." # # The "multicast bit" of a MAC address is defined to be "the least - # significant bit of the first octet". This worlds out to be the 41st bit + # significant bit of the first octet". This works out to be the 41st bit # counting from 1 being the least significant bit, or 1<<40. # # See https://en.wikipedia.org/wiki/MAC_address#Unicast_vs._multicast From 909ee79b0ff6f215167ea5af4694043ff8ae4b0b Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Sat, 25 Nov 2017 10:10:12 -0500 Subject: [PATCH 63/75] DEBUGGING TRAVIS - DO NOT MERGE --- Lib/uuid.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Lib/uuid.py b/Lib/uuid.py index 9e7c672f7a0a6f..c551bb60a0c9bd 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -374,7 +374,9 @@ def _find_mac(command, args, hw_identifiers, get_index): word = words[get_index(i)] mac = int(word.replace(b':', b''), 16) if _is_universal(mac): + print('%012x' % mac, 'universal', command) return mac + print('%012x' % mac, 'local', command) first_local_mac = first_local_mac or mac except (ValueError, IndexError): # Virtual interfaces, such as those provided by From 6acb120cca296749a8744bf726aa1c58cbc5741d Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Sun, 26 Nov 2017 11:44:25 -0500 Subject: [PATCH 64/75] Two tests cannot succeed on Travis-CI --- Lib/test/test_uuid.py | 4 ++++ Lib/uuid.py | 2 -- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index 97f34cdbe3310f..05427968692d26 100644 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -524,11 +524,15 @@ def check_node(self, node, requires=None, network=False): "%s is not an RFC 4122 node ID" % hex) @unittest.skipUnless(os.name == 'posix', 'requires Posix') + @unittest.skipIf(os.getenv('TRAVIS'), + 'Travis-CI has no universally administered MAC addresses') def test_ifconfig_getnode(self): node = self.uuid._ifconfig_getnode() self.check_node(node, 'ifconfig', True) @unittest.skipUnless(os.name == 'posix', 'requires Posix') + @unittest.skipIf(os.getenv('TRAVIS'), + 'Travis-CI has no universally administered MAC addresses') def test_ip_getnode(self): node = self.uuid._ip_getnode() self.check_node(node, 'ip', True) diff --git a/Lib/uuid.py b/Lib/uuid.py index c551bb60a0c9bd..9e7c672f7a0a6f 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -374,9 +374,7 @@ def _find_mac(command, args, hw_identifiers, get_index): word = words[get_index(i)] mac = int(word.replace(b':', b''), 16) if _is_universal(mac): - print('%012x' % mac, 'universal', command) return mac - print('%012x' % mac, 'local', command) first_local_mac = first_local_mac or mac except (ValueError, IndexError): # Virtual interfaces, such as those provided by From 18a96ce3df418755a670a34e1c42ce3c8529ee2b Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Tue, 21 Nov 2017 12:00:21 -0500 Subject: [PATCH 65/75] bpo-32107 - Fix test_uuid The original check was erroneous in two ways. First, the documentation said that "47 bit will never be set in IEEE 802 addresses obtained from network cards". I think this is referring to the universally vs. locally administered MAC addresses. Network cards from hardware manufacturers will always be universally administered in order to guarantee global uniqueness. But from https://en.wikipedia.org/wiki/MAC_address it says that this flag is the "distinguished by setting the second-least-significant bit of the first octet of the address", where a 0 means universally administered and a 1 means it's locally administered. This works out to the 42nd bit when counting the LSB as bit 1, or 1<<41. The second bug is that the original bitmask value isn't right for either description: % ./python.exe -c "from math import log2; print(log2(0x010000000000))" 40.0 This causes the test to fail on valid MAC addresses. Fix this by improving the comment, with references, and using the correct bitmask. --- Lib/test/test_uuid.py | 13 ++++++++++--- Lib/uuid.py | 3 +-- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index 083c2aa8aab54d..5adba57e07d9f2 100644 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -519,9 +519,16 @@ def check_node(self, node, requires=None, network=False): if support.verbose >= 2: print(hex, end=' ') if network: - # 47 bit will never be set in IEEE 802 addresses obtained - # from network cards. - self.assertFalse(node & 0x010000000000, hex) + # The second least significant bit of the first octet signifies + # whether the MAC (IEEE 802, or EUI-48) address is universally (0) + # or locally (1) administered. Network cards from hardware + # manufacturers will always be universally administered to + # guarantee global uniqueness of the MAC address. This bit works + # out to be the 42nd bit counting from 1 being the least + # significant, or 1<<41. For a good, simple explanation, see the + # section on Universal vs. local in this page: + # https://en.wikipedia.org/wiki/MAC_address + self.assertFalse(node & (1 << 41), hex) self.assertTrue(0 < node < (1 << 48), "%s is not an RFC 4122 node ID" % hex) diff --git a/Lib/uuid.py b/Lib/uuid.py index 020c6e73c863d4..fee809dd1abfab 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -404,8 +404,7 @@ def _arp_getnode(): # This works on Linux, FreeBSD and NetBSD mac = _find_mac('arp', '-an', [os.fsencode('(%s)' % ip_addr)], lambda i: i+2) - if mac: - return mac + return mac def _lanscan_getnode(): """Get the hardware address on Unix by running lanscan.""" From 83dc3a3b8ec25dcc70d97258827d3d0a00c77838 Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Tue, 21 Nov 2017 14:10:40 -0500 Subject: [PATCH 66/75] Return None instead of 0 --- Lib/uuid.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/uuid.py b/Lib/uuid.py index fee809dd1abfab..27335f89192909 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -404,7 +404,8 @@ def _arp_getnode(): # This works on Linux, FreeBSD and NetBSD mac = _find_mac('arp', '-an', [os.fsencode('(%s)' % ip_addr)], lambda i: i+2) - return mac + # Return None instead of 0. + return mac if mac else None def _lanscan_getnode(): """Get the hardware address on Unix by running lanscan.""" From fb156f3a5d46b92e5b79377936cd7225a4ffefe5 Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Tue, 21 Nov 2017 23:08:06 -0500 Subject: [PATCH 67/75] Never return a locally administered MAC address Also, clean up some return sites. --- Lib/test/test_uuid.py | 9 --------- Lib/uuid.py | 45 ++++++++++++++++++++++++++++++++++--------- 2 files changed, 36 insertions(+), 18 deletions(-) diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index 5adba57e07d9f2..19564f2d548d28 100644 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -519,15 +519,6 @@ def check_node(self, node, requires=None, network=False): if support.verbose >= 2: print(hex, end=' ') if network: - # The second least significant bit of the first octet signifies - # whether the MAC (IEEE 802, or EUI-48) address is universally (0) - # or locally (1) administered. Network cards from hardware - # manufacturers will always be universally administered to - # guarantee global uniqueness of the MAC address. This bit works - # out to be the 42nd bit counting from 1 being the least - # significant, or 1<<41. For a good, simple explanation, see the - # section on Universal vs. local in this page: - # https://en.wikipedia.org/wiki/MAC_address self.assertFalse(node & (1 << 41), hex) self.assertTrue(0 < node < (1 << 48), "%s is not an RFC 4122 node ID" % hex) diff --git a/Lib/uuid.py b/Lib/uuid.py index 27335f89192909..0535433f2bffda 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -342,11 +342,26 @@ def _popen(command, *args): env=env) return proc +# For MAC (a.k.a. IEEE 802, or EUI-48) addresses, the second least significant +# bit of the first octet signifies whether the MAC address is universally (0) +# or locally (1) administered. Network cards from hardware manufacturers will +# always be universally administered to guarantee global uniqueness of the MAC +# address, but any particular machine may have other interfaces which are +# locally administered. An example of the latter is the bridge interface to +# the Touch Bar on MacBook Pros. +# +# This bit works out to be the 42nd bit counting from 1 being the least +# significant, or 1<<41. We'll skip over any locally administered MAC +# addresses, as it makes no sense to use those in UUID calculation. +# +# For a good, simple explanation, see the section on Universal vs. local in +# this page: https://en.wikipedia.org/wiki/MAC_address + def _find_mac(command, args, hw_identifiers, get_index): try: proc = _popen(command, *args.split()) if not proc: - return + return None with proc: for line in proc.stdout: words = line.lower().rstrip().split() @@ -355,7 +370,7 @@ def _find_mac(command, args, hw_identifiers, get_index): try: word = words[get_index(i)] mac = int(word.replace(b':', b''), 16) - if mac: + if ~(mac & (1<<41)): return mac except (ValueError, IndexError): # Virtual interfaces, such as those provided by @@ -366,6 +381,7 @@ def _find_mac(command, args, hw_identifiers, get_index): pass except OSError: pass + return None def _ifconfig_getnode(): """Get the hardware address on Unix by running ifconfig.""" @@ -375,6 +391,7 @@ def _ifconfig_getnode(): mac = _find_mac('ifconfig', args, keywords, lambda i: i+1) if mac: return mac + return None def _ip_getnode(): """Get the hardware address on Unix by running ip.""" @@ -382,6 +399,7 @@ def _ip_getnode(): mac = _find_mac('ip', 'link list', [b'link/ether'], lambda i: i+1) if mac: return mac + return None def _arp_getnode(): """Get the hardware address on Unix by running arp.""" @@ -405,7 +423,9 @@ def _arp_getnode(): mac = _find_mac('arp', '-an', [os.fsencode('(%s)' % ip_addr)], lambda i: i+2) # Return None instead of 0. - return mac if mac else None + if mac: + return mac + return None def _lanscan_getnode(): """Get the hardware address on Unix by running lanscan.""" @@ -418,25 +438,26 @@ def _netstat_getnode(): try: proc = _popen('netstat', '-ia') if not proc: - return + return None with proc: words = proc.stdout.readline().rstrip().split() try: i = words.index(b'Address') except ValueError: - return + return None for line in proc.stdout: try: words = line.rstrip().split() word = words[i] if len(word) == 17 and word.count(b':') == 5: mac = int(word.replace(b':', b''), 16) - if mac: + if ~(mac & (1<<41)): return mac except (ValueError, IndexError): pass except OSError: pass + return None def _ipconfig_getnode(): """Get the hardware address on Windows by running ipconfig.exe.""" @@ -458,7 +479,10 @@ def _ipconfig_getnode(): for line in pipe: value = line.split(':')[-1].strip().lower() if re.match('([0-9a-f][0-9a-f]-){5}[0-9a-f][0-9a-f]', value): - return int(value.replace('-', ''), 16) + mac = int(value.replace('-', ''), 16) + if ~(mac & (1<<41)): + return mac + return None def _netbios_getnode(): """Get the hardware address on Windows using NetBIOS calls. @@ -469,7 +493,7 @@ def _netbios_getnode(): ncb.Buffer = adapters = netbios.LANA_ENUM() adapters._pack() if win32wnet.Netbios(ncb) != 0: - return + return None adapters._unpack() for i in range(adapters.length): ncb.Reset() @@ -488,7 +512,10 @@ def _netbios_getnode(): bytes = status.adapter_address[:6] if len(bytes) != 6: continue - return int.from_bytes(bytes, 'big') + mac = int.from_bytes(bytes, 'big') + if ~(mac & (1<<41)): + return mac + return None _generate_time_safe = _UuidCreate = None From 3e8e17d6c38dc1abc9812e6985fd55f4a09b769f Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Thu, 23 Nov 2017 10:16:21 -0500 Subject: [PATCH 68/75] Several sleep-improved fixes * Add a helper function to determine whether a MAC address is universally administered or not. * In the various MAC getter functions, keep track of the first locally administered MAC, and if no universally administered MAC is found, return the first local MAC. * In the ultimate fallback _random_getnode(), add a comment for the multicast bit being set, and use a better bitmask * In getnode(), just fall back to _random_getnode() explicitly if no other MAC has been found. --- Lib/test/test_uuid.py | 2 +- Lib/uuid.py | 45 +++++++++++++++++++++++++++++++------------ 2 files changed, 34 insertions(+), 13 deletions(-) diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index 19564f2d548d28..97f34cdbe3310f 100644 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -562,7 +562,7 @@ def test_netbios_getnode(self): def test_random_getnode(self): node = self.uuid._random_getnode() # Least significant bit of first octet must be set. - self.assertTrue(node & 0x010000000000, '%012x' % node) + self.assertTrue(node & (1 << 40), '%012x' % node) self.check_node(node) @unittest.skipUnless(os.name == 'posix', 'requires Posix') diff --git a/Lib/uuid.py b/Lib/uuid.py index 0535433f2bffda..3eec69f6f99d4b 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -354,10 +354,13 @@ def _popen(command, *args): # significant, or 1<<41. We'll skip over any locally administered MAC # addresses, as it makes no sense to use those in UUID calculation. # -# For a good, simple explanation, see the section on Universal vs. local in -# this page: https://en.wikipedia.org/wiki/MAC_address +# See https://en.wikipedia.org/wiki/MAC_address#Universal_vs._local + +def _is_universal(mac): + return not (mac & (1 << 41)) def _find_mac(command, args, hw_identifiers, get_index): + first_local_mac = None try: proc = _popen(command, *args.split()) if not proc: @@ -370,8 +373,9 @@ def _find_mac(command, args, hw_identifiers, get_index): try: word = words[get_index(i)] mac = int(word.replace(b':', b''), 16) - if ~(mac & (1<<41)): + if _is_universal(mac): return mac + first_local_mac = first_local_mac or mac except (ValueError, IndexError): # Virtual interfaces, such as those provided by # VPNs, do not have a colon-delimited MAC address @@ -381,7 +385,7 @@ def _find_mac(command, args, hw_identifiers, get_index): pass except OSError: pass - return None + return first_local_mac or None def _ifconfig_getnode(): """Get the hardware address on Unix by running ifconfig.""" @@ -435,6 +439,7 @@ def _lanscan_getnode(): def _netstat_getnode(): """Get the hardware address on Unix by running netstat.""" # This might work on AIX, Tru64 UNIX. + first_local_mac = None try: proc = _popen('netstat', '-ia') if not proc: @@ -451,17 +456,19 @@ def _netstat_getnode(): word = words[i] if len(word) == 17 and word.count(b':') == 5: mac = int(word.replace(b':', b''), 16) - if ~(mac & (1<<41)): + if _is_universal(mac): return mac + first_local_mac = first_local_mac or mac except (ValueError, IndexError): pass except OSError: pass - return None + return first_local_mac or None def _ipconfig_getnode(): """Get the hardware address on Windows by running ipconfig.exe.""" import os, re + first_local_mac = None dirs = ['', r'c:\windows\system32', r'c:\winnt\system32'] try: import ctypes @@ -480,14 +487,16 @@ def _ipconfig_getnode(): value = line.split(':')[-1].strip().lower() if re.match('([0-9a-f][0-9a-f]-){5}[0-9a-f][0-9a-f]', value): mac = int(value.replace('-', ''), 16) - if ~(mac & (1<<41)): + if _is_universal(mac): return mac - return None + first_local_mac = first_local_mac or mac + return first_local_mac or None def _netbios_getnode(): """Get the hardware address on Windows using NetBIOS calls. See http://support.microsoft.com/kb/118623 for details.""" import win32wnet, netbios + first_local_mac = None ncb = netbios.NCB() ncb.Command = netbios.NCBENUM ncb.Buffer = adapters = netbios.LANA_ENUM() @@ -513,8 +522,9 @@ def _netbios_getnode(): if len(bytes) != 6: continue mac = int.from_bytes(bytes, 'big') - if ~(mac & (1<<41)): + if _is_universal(mac): return mac + first_local_mac = first_local_mac or mac return None @@ -628,9 +638,19 @@ def _windll_getnode(): return UUID(bytes=bytes_(_buffer.raw)).node def _random_getnode(): - """Get a random node ID, with eighth bit set as suggested by RFC 4122.""" + """Get a random node ID.""" + # RFC 4122, $4.1.6. says "For systems with no IEEE address, a randomly or + # pseudo-randomly generated value may be used; see Section 4.5. The + # multicast bit must be set in such addresses, in order that they will + # never conflict with addresses obtained from network cards." + # + # The "multicast bit" of a MAC address is defined to be "the least + # significant bit of the first octet". This worlds out to be the 41st bit + # counting from 1 being the least significant bit, or 1<<40. + # + # See https://en.wikipedia.org/wiki/MAC_address#Unicast_vs._multicast import random - return random.getrandbits(48) | 0x010000000000 + return random.getrandbits(48) | (1 << 40) _node = None @@ -653,13 +673,14 @@ def getnode(): getters = [_unix_getnode, _ifconfig_getnode, _ip_getnode, _arp_getnode, _lanscan_getnode, _netstat_getnode] - for getter in getters + [_random_getnode]: + for getter in getters: try: _node = getter() except: continue if _node is not None: return _node + return _random_getnode() _last_timestamp = None From 668f4f6db80145cf9abb768dd2b01b8d8c15945c Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Thu, 23 Nov 2017 10:25:51 -0500 Subject: [PATCH 69/75] A missed return --- Lib/uuid.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/uuid.py b/Lib/uuid.py index 3eec69f6f99d4b..364b5b0272b785 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -525,7 +525,7 @@ def _netbios_getnode(): if _is_universal(mac): return mac first_local_mac = first_local_mac or mac - return None + return first_local_mac or None _generate_time_safe = _UuidCreate = None From 6eecb80b85084c81d21aa9c276b5fc90f8b659e1 Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Thu, 23 Nov 2017 10:27:21 -0500 Subject: [PATCH 70/75] Two typos --- Lib/uuid.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/uuid.py b/Lib/uuid.py index 364b5b0272b785..9e7c672f7a0a6f 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -639,13 +639,13 @@ def _windll_getnode(): def _random_getnode(): """Get a random node ID.""" - # RFC 4122, $4.1.6. says "For systems with no IEEE address, a randomly or + # RFC 4122, $4.1.6 says "For systems with no IEEE address, a randomly or # pseudo-randomly generated value may be used; see Section 4.5. The # multicast bit must be set in such addresses, in order that they will # never conflict with addresses obtained from network cards." # # The "multicast bit" of a MAC address is defined to be "the least - # significant bit of the first octet". This worlds out to be the 41st bit + # significant bit of the first octet". This works out to be the 41st bit # counting from 1 being the least significant bit, or 1<<40. # # See https://en.wikipedia.org/wiki/MAC_address#Unicast_vs._multicast From 87712f3e89580912773a38c7ba6d54b645f75339 Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Sat, 25 Nov 2017 10:10:12 -0500 Subject: [PATCH 71/75] DEBUGGING TRAVIS - DO NOT MERGE --- Lib/uuid.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Lib/uuid.py b/Lib/uuid.py index 9e7c672f7a0a6f..c551bb60a0c9bd 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -374,7 +374,9 @@ def _find_mac(command, args, hw_identifiers, get_index): word = words[get_index(i)] mac = int(word.replace(b':', b''), 16) if _is_universal(mac): + print('%012x' % mac, 'universal', command) return mac + print('%012x' % mac, 'local', command) first_local_mac = first_local_mac or mac except (ValueError, IndexError): # Virtual interfaces, such as those provided by From 740393ebf3d1444259274f96a8f660292909ccfa Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Sun, 26 Nov 2017 11:44:25 -0500 Subject: [PATCH 72/75] Two tests cannot succeed on Travis-CI --- Lib/test/test_uuid.py | 4 ++++ Lib/uuid.py | 2 -- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index 97f34cdbe3310f..05427968692d26 100644 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -524,11 +524,15 @@ def check_node(self, node, requires=None, network=False): "%s is not an RFC 4122 node ID" % hex) @unittest.skipUnless(os.name == 'posix', 'requires Posix') + @unittest.skipIf(os.getenv('TRAVIS'), + 'Travis-CI has no universally administered MAC addresses') def test_ifconfig_getnode(self): node = self.uuid._ifconfig_getnode() self.check_node(node, 'ifconfig', True) @unittest.skipUnless(os.name == 'posix', 'requires Posix') + @unittest.skipIf(os.getenv('TRAVIS'), + 'Travis-CI has no universally administered MAC addresses') def test_ip_getnode(self): node = self.uuid._ip_getnode() self.check_node(node, 'ip', True) diff --git a/Lib/uuid.py b/Lib/uuid.py index c551bb60a0c9bd..9e7c672f7a0a6f 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -374,9 +374,7 @@ def _find_mac(command, args, hw_identifiers, get_index): word = words[get_index(i)] mac = int(word.replace(b':', b''), 16) if _is_universal(mac): - print('%012x' % mac, 'universal', command) return mac - print('%012x' % mac, 'local', command) first_local_mac = first_local_mac or mac except (ValueError, IndexError): # Virtual interfaces, such as those provided by From 225b0638b58b9e94b75377b126467b1295a48946 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 26 Nov 2017 22:42:57 +0200 Subject: [PATCH 73/75] bpo-32107: Remove the check of the multicast bit in test_uuid. While it is required that the multicast bit (1<<40) should be set in random generated addresses, there is no requirement that it should be cleared in addresses obtained from network cards. --- Lib/test/test_uuid.py | 39 +++++++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index 05427968692d26..ca00bdb13534c1 100644 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -512,62 +512,69 @@ def test_find_mac(self): self.assertEqual(mac, 0x1234567890ab) - def check_node(self, node, requires=None, network=False): + def check_node(self, node, requires=None, *, random=False): if requires and node is None: self.skipTest('requires ' + requires) hex = '%012x' % node if support.verbose >= 2: print(hex, end=' ') - if network: - self.assertFalse(node & (1 << 41), hex) + # The MAC address will be universally administered (i.e. the second + # least significant bit of the first octet must be unset) for any + # physical interface, such as an ethernet port or wireless adapter. + # There are some cases where this won't be the case. Randomly + # generated MACs may not be universally administered, but they must + # have their multicast bit set, though this is tested in the + # `test_random_getnode()` method specifically. Another case is the + # Travis-CI case, which apparently only has locally administered MAC + # addresses. + if not random and not os.getenv('TRAVIS'): + self.assertFalse(node & (1 << 41), '%012x' % node) self.assertTrue(0 < node < (1 << 48), "%s is not an RFC 4122 node ID" % hex) @unittest.skipUnless(os.name == 'posix', 'requires Posix') - @unittest.skipIf(os.getenv('TRAVIS'), - 'Travis-CI has no universally administered MAC addresses') def test_ifconfig_getnode(self): node = self.uuid._ifconfig_getnode() - self.check_node(node, 'ifconfig', True) + self.check_node(node, 'ifconfig') @unittest.skipUnless(os.name == 'posix', 'requires Posix') - @unittest.skipIf(os.getenv('TRAVIS'), - 'Travis-CI has no universally administered MAC addresses') def test_ip_getnode(self): node = self.uuid._ip_getnode() - self.check_node(node, 'ip', True) + self.check_node(node, 'ip') @unittest.skipUnless(os.name == 'posix', 'requires Posix') def test_arp_getnode(self): node = self.uuid._arp_getnode() - self.check_node(node, 'arp', True) + self.check_node(node, 'arp') @unittest.skipUnless(os.name == 'posix', 'requires Posix') def test_lanscan_getnode(self): node = self.uuid._lanscan_getnode() - self.check_node(node, 'lanscan', True) + self.check_node(node, 'lanscan') @unittest.skipUnless(os.name == 'posix', 'requires Posix') def test_netstat_getnode(self): node = self.uuid._netstat_getnode() - self.check_node(node, 'netstat', True) + self.check_node(node, 'netstat') @unittest.skipUnless(os.name == 'nt', 'requires Windows') def test_ipconfig_getnode(self): node = self.uuid._ipconfig_getnode() - self.check_node(node, 'ipconfig', True) + self.check_node(node, 'ipconfig') @unittest.skipUnless(importable('win32wnet'), 'requires win32wnet') @unittest.skipUnless(importable('netbios'), 'requires netbios') def test_netbios_getnode(self): node = self.uuid._netbios_getnode() - self.check_node(node, network=True) + self.check_node(node) def test_random_getnode(self): node = self.uuid._random_getnode() - # Least significant bit of first octet must be set. + # The multicast bit, i.e. the least significant bit of first octet, + # must be set for randomly generated MAC addresses. See RFC 4122, + # $4.1.6. self.assertTrue(node & (1 << 40), '%012x' % node) - self.check_node(node) + self.check_node(node, random=True) @unittest.skipUnless(os.name == 'posix', 'requires Posix') def test_unix_getnode(self): From 5b8ca9056d5a8e27d972fd53d542091880a70c77 Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Sat, 25 Nov 2017 10:10:12 -0500 Subject: [PATCH 74/75] DEBUGGING TRAVIS - DO NOT MERGE --- Lib/uuid.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Lib/uuid.py b/Lib/uuid.py index 9e7c672f7a0a6f..c551bb60a0c9bd 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -374,7 +374,9 @@ def _find_mac(command, args, hw_identifiers, get_index): word = words[get_index(i)] mac = int(word.replace(b':', b''), 16) if _is_universal(mac): + print('%012x' % mac, 'universal', command) return mac + print('%012x' % mac, 'local', command) first_local_mac = first_local_mac or mac except (ValueError, IndexError): # Virtual interfaces, such as those provided by From 2aa4f5082517aff7616c6bf792211ccb8aec443d Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Sun, 26 Nov 2017 11:44:25 -0500 Subject: [PATCH 75/75] Two tests cannot succeed on Travis-CI --- Lib/test/test_uuid.py | 4 ++++ Lib/uuid.py | 2 -- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index ca00bdb13534c1..00a69158ec3ad4 100644 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -533,11 +533,15 @@ def check_node(self, node, requires=None, *, random=False): "%s is not an RFC 4122 node ID" % hex) @unittest.skipUnless(os.name == 'posix', 'requires Posix') + @unittest.skipIf(os.getenv('TRAVIS'), + 'Travis-CI has no universally administered MAC addresses') def test_ifconfig_getnode(self): node = self.uuid._ifconfig_getnode() self.check_node(node, 'ifconfig') @unittest.skipUnless(os.name == 'posix', 'requires Posix') + @unittest.skipIf(os.getenv('TRAVIS'), + 'Travis-CI has no universally administered MAC addresses') def test_ip_getnode(self): node = self.uuid._ip_getnode() self.check_node(node, 'ip') diff --git a/Lib/uuid.py b/Lib/uuid.py index c551bb60a0c9bd..9e7c672f7a0a6f 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -374,9 +374,7 @@ def _find_mac(command, args, hw_identifiers, get_index): word = words[get_index(i)] mac = int(word.replace(b':', b''), 16) if _is_universal(mac): - print('%012x' % mac, 'universal', command) return mac - print('%012x' % mac, 'local', command) first_local_mac = first_local_mac or mac except (ValueError, IndexError): # Virtual interfaces, such as those provided by