From 4f310f4a516ca4d05bbc832779af4dc54b01b68a Mon Sep 17 00:00:00 2001 From: lasers Date: Thu, 25 Jan 2018 14:21:34 -0600 Subject: [PATCH 001/182] docs: fix battery variable (#1212) --- docs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index d58e1c19d..4ad681e3e 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -759,7 +759,7 @@ Sensors >>> battery = psutil.sensors_battery() >>> battery sbattery(percent=93, secsleft=16628, power_plugged=False) - >>> print("charge = %s%%, time left = %s" % (batt.percent, secs2hours(batt.secsleft))) + >>> print("charge = %s%%, time left = %s" % (battery.percent, secs2hours(battery.secsleft))) charge = 93%, time left = 4:37:08 See also `battery.py `__ and `sensors.py `__ for an example application. From 95155afe90820c4c012130001fe86713f6a13253 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 2 Feb 2018 15:42:46 +0100 Subject: [PATCH 002/182] add StackOverflow link --- README.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/README.rst b/README.rst index 4ac707aba..48b1411f8 100644 --- a/README.rst +++ b/README.rst @@ -35,6 +35,7 @@ Quick links - `Documentation `_ - `Download `_ - `Forum `_ +- `StackOverflow `_ - `Blog `_ - `Development guide `_ - `What's new `_ From 7618de9683684811402f50b08ca3f3e979774e72 Mon Sep 17 00:00:00 2001 From: Dan Vinakovsky Date: Mon, 12 Feb 2018 18:29:34 -0500 Subject: [PATCH 003/182] fix compatibility with python 2.6.x (#1216) --- psutil/_pswindows.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/psutil/_pswindows.py b/psutil/_pswindows.py index b6c58c936..0eb4b14f8 100644 --- a/psutil/_pswindows.py +++ b/psutil/_pswindows.py @@ -200,7 +200,7 @@ def py2_strencode(s): if isinstance(s, str): return s else: - return s.encode(ENCODING, errors=ENCODING_ERRS) + return s.encode(ENCODING, ENCODING_ERRS) # ===================================================================== From 62e1783764420f36cd270135decb00b324add7fe Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 13 Feb 2018 00:31:59 +0100 Subject: [PATCH 004/182] #1216 - give credit to @hexaclock --- CREDITS | 4 ++++ HISTORY.rst | 9 +++++++++ 2 files changed, 13 insertions(+) diff --git a/CREDITS b/CREDITS index cf9ce493c..798b36e39 100644 --- a/CREDITS +++ b/CREDITS @@ -515,3 +515,7 @@ I: 1167 N: janderbrain W: https://github.com/janderbrain I: 1169 + +N: Dan Vinakovsky +W: https://github.com/hexaclock +I: 1216 diff --git a/HISTORY.rst b/HISTORY.rst index dd813c5b5..463860c25 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,5 +1,14 @@ *Bug tracker at https://github.com/giampaolo/psutil/issues* +5.4.4 +===== + +XXXX-XX-XX + +**Bug fixes** + +- 1216_: fix compatibility with python 2.6 on Windows (patch by Dan Vinakovsky) + 5.4.3 ===== From d690ae6283b996ffd2098e47652bf84cbd7e87bf Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 14 Feb 2018 14:02:37 +0100 Subject: [PATCH 005/182] rename fun --- psutil/tests/__init__.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 9e8d8596b..499240fb2 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -283,7 +283,7 @@ def stop(self): # =================================================================== -def _cleanup_on_err(fun): +def _reap_children_on_err(fun): @functools.wraps(fun) def wrapper(*args, **kwargs): try: @@ -294,7 +294,7 @@ def wrapper(*args, **kwargs): return wrapper -@_cleanup_on_err +@_reap_children_on_err def get_test_subprocess(cmd=None, **kwds): """Creates a python subprocess which does nothing for 60 secs and return it as subprocess.Popen instance. @@ -327,7 +327,7 @@ def get_test_subprocess(cmd=None, **kwds): return sproc -@_cleanup_on_err +@_reap_children_on_err def create_proc_children_pair(): """Create a subprocess which creates another one as in: A (us) -> B (child) -> C (grandchild). @@ -399,7 +399,7 @@ def create_zombie_proc(): conn.close() -@_cleanup_on_err +@_reap_children_on_err def pyrun(src, **kwds): """Run python 'src' code string in a separate interpreter. Returns a subprocess.Popen instance. @@ -416,7 +416,7 @@ def pyrun(src, **kwds): return subp -@_cleanup_on_err +@_reap_children_on_err def sh(cmd, **kwds): """run cmd in a subprocess and return its output. raises RuntimeError on error. From dd19ba7b5bbf5ea67a4d386baa3fabd13b118533 Mon Sep 17 00:00:00 2001 From: Thijs Triemstra Date: Tue, 20 Feb 2018 13:18:24 +0100 Subject: [PATCH 006/182] enable pip cache in appveyor build (#1221) --- appveyor.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/appveyor.yml b/appveyor.yml index 092dc23a8..28f5f7f6f 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -91,6 +91,9 @@ after_test: artifacts: - path: dist\* +cache: + - '%LOCALAPPDATA%\pip\Cache' + # on_success: # - might want to upload the content of dist/*.whl to a public wheelhouse From 29732d0476efec178a5b09bc623b13df89a7936f Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 23 Feb 2018 22:48:39 +0100 Subject: [PATCH 007/182] add test for safe_mkdir --- psutil/tests/test_misc.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index f67c0e4cd..1d9067e77 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -54,6 +54,7 @@ from psutil.tests import retry from psutil.tests import ROOT_DIR from psutil.tests import run_test_module_by_name +from psutil.tests import safe_mkdir from psutil.tests import safe_rmpath from psutil.tests import SCRIPTS_DIR from psutil.tests import sh @@ -895,6 +896,12 @@ def setUp(self): tearDown = setUp + def test_safe_mkdir(self): + safe_mkdir(TESTFN) + assert os.path.isdir(TESTFN) + safe_mkdir(TESTFN) + assert os.path.isdir(TESTFN) + def test_safe_rmpath(self): # test file is removed open(TESTFN, 'w').close() From 053a97f7fc70c47410d4ad2a43c284689181d84a Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 1 Mar 2018 11:10:29 +0100 Subject: [PATCH 008/182] update doc + bytes conversion example --- docs/index.rst | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/docs/index.rst b/docs/index.rst index 4ad681e3e..10f0dfb08 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -2525,6 +2525,38 @@ Top 3 processes opening more file descriptors:: (2721, {'name': 'chrome', 'num_fds': 185}), (2650, {'name': 'chrome', 'num_fds': 354})] +Bytes conversion +---------------- + +:: + + import psutil + + def bytes2human(n): + # http://code.activestate.com/recipes/578019 + # >>> bytes2human(10000) + # '9.8K' + # >>> bytes2human(100001221) + # '95.4M' + symbols = ('K', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y') + prefix = {} + for i, s in enumerate(symbols): + prefix[s] = 1 << (i + 1) * 10 + for s in reversed(symbols): + if n >= prefix[s]: + value = float(n) / prefix[s] + return '%.1f%s' % (value, s) + return "%sB" % n + + total = psutil.disk_usage('/').total + print(total) + print(bytes2human(total)) + +...prints:: + + 100399730688 + 93.5G + FAQs ==== From 74248cc706bdf01f80d1c704a094ff30974cfa93 Mon Sep 17 00:00:00 2001 From: Jon Dufresne Date: Sun, 4 Mar 2018 09:44:10 -0800 Subject: [PATCH 009/182] Remove additional workarounds and support for Python 3.2 & 3.3 (#1228) Support for Python 3.3 was dropped in version 5.4.1. Support for Python 3.2 was dropped earlier. Remove all references to these unsupported versions including documentation, scripts, workarounds, etc. Eases maintenance as fewer workarounds are used for unsupported environments. --- .ci/travis/install.sh | 12 ------------ INSTALL.rst | 2 +- make.bat | 4 ++-- psutil/__init__.py | 1 - psutil/_compat.py | 10 +--------- psutil/tests/__main__.py | 2 -- psutil/tests/test_contracts.py | 1 - psutil/tests/test_posix.py | 1 - psutil/tests/test_windows.py | 1 - tox.ini | 5 +---- 10 files changed, 5 insertions(+), 34 deletions(-) diff --git a/.ci/travis/install.sh b/.ci/travis/install.sh index 677dc4653..bb86700ea 100755 --- a/.ci/travis/install.sh +++ b/.ci/travis/install.sh @@ -24,14 +24,6 @@ if [[ "$(uname -s)" == 'Darwin' ]]; then pyenv install 2.7.10 pyenv virtualenv 2.7.10 psutil ;; - # py32) - # pyenv install 3.2.6 - # pyenv virtualenv 3.2.6 psutil - # ;; - # py33) - # pyenv install 3.3.6 - # pyenv virtualenv 3.3.6 psutil - # ;; py34) pyenv install 3.4.3 pyenv virtualenv 3.4.3 psutil @@ -45,10 +37,6 @@ if [[ $TRAVIS_PYTHON_VERSION == '2.6' ]] || [[ $PYVER == 'py26' ]]; then pip install -U ipaddress unittest2 argparse mock==1.0.1 elif [[ $TRAVIS_PYTHON_VERSION == '2.7' ]] || [[ $PYVER == 'py27' ]]; then pip install -U ipaddress mock -elif [[ $TRAVIS_PYTHON_VERSION == '3.2' ]] || [[ $PYVER == 'py32' ]]; then - pip install -U ipaddress mock -elif [[ $TRAVIS_PYTHON_VERSION == '3.3' ]] || [[ $PYVER == 'py33' ]]; then - pip install -U ipaddress fi pip install -U coverage coveralls flake8 pep8 setuptools diff --git a/INSTALL.rst b/INSTALL.rst index 8d54d6ae1..9e5defb42 100644 --- a/INSTALL.rst +++ b/INSTALL.rst @@ -89,7 +89,7 @@ This `blog post `__ -* Python 3.3, 3.4: `VS-2010 `__ +* Python 3.4: `VS-2010 `__ * Python 3.5+: `VS-2015 `__ Compiling 64 bit versions of Python 2.6 and 2.7 with VS 2008 requires diff --git a/make.bat b/make.bat index cdabe3a61..c7c319026 100644 --- a/make.bat +++ b/make.bat @@ -7,8 +7,8 @@ rem psutil ("make.bat build", "make.bat install") and running tests rem ("make.bat test"). rem rem This script is modeled after my Windows installation which uses: -rem - Visual studio 2008 for Python 2.6, 2.7, 3.2 -rem - Visual studio 2010 for Python 3.3+ +rem - Visual studio 2008 for Python 2.6, 2.7 +rem - Visual studio 2010 for Python 3.4+ rem ...therefore it might not work on your Windows installation. rem rem By default C:\Python27\python.exe is used. diff --git a/psutil/__init__.py b/psutil/__init__.py index 5e29a7fc5..97767f116 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -43,7 +43,6 @@ from ._common import memoize from ._common import memoize_when_activated from ._common import wrap_numbers as _wrap_numbers -from ._compat import callable from ._compat import long from ._compat import PY3 as _PY3 diff --git a/psutil/_compat.py b/psutil/_compat.py index de91638f6..08aefe4b7 100644 --- a/psutil/_compat.py +++ b/psutil/_compat.py @@ -10,7 +10,7 @@ import sys __all__ = ["PY3", "long", "xrange", "unicode", "basestring", "u", "b", - "callable", "lru_cache", "which"] + "lru_cache", "which"] PY3 = sys.version_info[0] == 3 @@ -38,14 +38,6 @@ def b(s): return s -# removed in 3.0, reintroduced in 3.2 -try: - callable = callable -except NameError: - def callable(obj): - return any("__call__" in klass.__dict__ for klass in type(obj).__mro__) - - # --- stdlib additions diff --git a/psutil/tests/__main__.py b/psutil/tests/__main__.py index 2cdf5c425..62fe07428 100755 --- a/psutil/tests/__main__.py +++ b/psutil/tests/__main__.py @@ -32,8 +32,6 @@ TEST_DEPS.extend(["ipaddress", "unittest2", "argparse", "mock==1.0.1"]) elif sys.version_info[:2] == (2, 7) or sys.version_info[:2] <= (3, 2): TEST_DEPS.extend(["ipaddress", "mock"]) -elif sys.version_info[:2] == (3, 3): - TEST_DEPS.extend(["ipaddress"]) def install_pip(): diff --git a/psutil/tests/test_contracts.py b/psutil/tests/test_contracts.py index 855b53bf9..0dfb3e2a8 100755 --- a/psutil/tests/test_contracts.py +++ b/psutil/tests/test_contracts.py @@ -27,7 +27,6 @@ from psutil import POSIX from psutil import SUNOS from psutil import WINDOWS -from psutil._compat import callable from psutil._compat import long from psutil.tests import bind_unix_socket from psutil.tests import check_connection_ntuple diff --git a/psutil/tests/test_posix.py b/psutil/tests/test_posix.py index c59f9a1c7..e9a6f5f63 100755 --- a/psutil/tests/test_posix.py +++ b/psutil/tests/test_posix.py @@ -23,7 +23,6 @@ from psutil import OSX from psutil import POSIX from psutil import SUNOS -from psutil._compat import callable from psutil._compat import PY3 from psutil.tests import APPVEYOR from psutil.tests import get_kernel_version diff --git a/psutil/tests/test_windows.py b/psutil/tests/test_windows.py index e4a719ea4..32c46f67a 100755 --- a/psutil/tests/test_windows.py +++ b/psutil/tests/test_windows.py @@ -21,7 +21,6 @@ import psutil from psutil import WINDOWS -from psutil._compat import callable from psutil.tests import APPVEYOR from psutil.tests import get_test_subprocess from psutil.tests import HAS_BATTERY diff --git a/tox.ini b/tox.ini index 3bee1d5a0..305d81cec 100644 --- a/tox.ini +++ b/tox.ini @@ -5,7 +5,7 @@ # directory. [tox] -envlist = py26, py27, py33, py34, py35, py36 +envlist = py26, py27, py34, py35, py36 [testenv] deps = @@ -15,9 +15,6 @@ deps = py26: unittest2 py27: ipaddress py27: mock - py32: ipaddress - py32: mock - py33: ipaddress setenv = PYTHONPATH = {toxinidir}/test From 2650852829e0006d89f57bffc44a20613f47ddea Mon Sep 17 00:00:00 2001 From: Jon Dufresne Date: Sun, 4 Mar 2018 10:04:44 -0800 Subject: [PATCH 010/182] Remove unnecessary tox PYTHONPATH configuration (#1229) No "test" directory exists. The configuration is unneeded. --- tox.ini | 1 - 1 file changed, 1 deletion(-) diff --git a/tox.ini b/tox.ini index 305d81cec..ab8174ca3 100644 --- a/tox.ini +++ b/tox.ini @@ -17,7 +17,6 @@ deps = py27: mock setenv = - PYTHONPATH = {toxinidir}/test TOX = 1 commands = From 301be5c1a07525e702186381ba7f81492007606f Mon Sep 17 00:00:00 2001 From: Vincent Vinet Date: Thu, 8 Mar 2018 14:18:04 -0500 Subject: [PATCH 011/182] doc typo (#1234) --- docs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index 10f0dfb08..4f77b2e49 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -849,7 +849,7 @@ Functions Sorting order in which processes are returned is based on their PID. *attrs* and *ad_value* have the same meaning as in :meth:`Process.as_dict()`. - If *attrs* is specified :meth:`Process.as_dict()` is called interanally and + If *attrs* is specified :meth:`Process.as_dict()` is called internally and the resulting dict is stored as a ``info`` attribute which is attached to the returned :class:`Process` instances. If *attrs* is an empty list it will retrieve all process info (slow). From f97547272a3588a236a3ba08112c8c8e943fece4 Mon Sep 17 00:00:00 2001 From: Jon Dufresne Date: Thu, 8 Mar 2018 11:18:23 -0800 Subject: [PATCH 012/182] Remove unused import 'psutil.tests' in test_unicode.py (#1231) --- psutil/tests/test_unicode.py | 1 - 1 file changed, 1 deletion(-) diff --git a/psutil/tests/test_unicode.py b/psutil/tests/test_unicode.py index c2a2f8479..6383c9bec 100755 --- a/psutil/tests/test_unicode.py +++ b/psutil/tests/test_unicode.py @@ -87,7 +87,6 @@ from psutil.tests import unittest from psutil.tests import unix_socket_path import psutil -import psutil.tests def safe_rmpath(path): From 98cce4a30e79540a635386a9acaf5070ee2b2757 Mon Sep 17 00:00:00 2001 From: Jon Dufresne Date: Thu, 8 Mar 2018 11:18:45 -0800 Subject: [PATCH 013/182] Use environment markers for conditional dependencies (#1230) Using environment markers has support among modern Python packaging tools (setuptools, pip, wheel, etc.). Allows describing environment specific dependencies in setup.py without using runtime logic. --- setup.py | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/setup.py b/setup.py index d8db694eb..61056f5f1 100755 --- a/setup.py +++ b/setup.py @@ -52,18 +52,6 @@ if POSIX: sources.append('psutil/_psutil_posix.c') -tests_require = [] -if sys.version_info[:2] <= (2, 6): - tests_require.append('unittest2') -if sys.version_info[:2] <= (2, 7): - tests_require.append('mock') -if sys.version_info[:2] <= (3, 2): - tests_require.append('ipaddress') - -extras_require = {} -if sys.version_info[:2] <= (3, 3): - extras_require.update(dict(enum='enum34')) - def get_version(): INIT = os.path.join(HERE, 'psutil/__init__.py') @@ -340,8 +328,14 @@ def main(): kwargs.update( python_requires=">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*", test_suite="psutil.tests.get_suite", - tests_require=tests_require, - extras_require=extras_require, + tests_require=[ + 'ipaddress; python_version < "3.3"', + 'mock; python_version < "3.3"', + 'unittest2; python_version < "2.7"', + ], + extras_require={ + 'enum': 'enum34; python_version < "3.4"', + }, zip_safe=False, ) setup(**kwargs) From ccf54249ded48599fe6ccc259f3a78d1d67aafc2 Mon Sep 17 00:00:00 2001 From: Jon Dufresne Date: Thu, 8 Mar 2018 11:19:54 -0800 Subject: [PATCH 014/182] Be more explicit about supported Python version (#1233) Python 3.0, 3.1, 3.2, and 3.3 are not supported. Now phrased as: "Python versions 2.6, 2.7, and 3.4+". --- README.rst | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 48b1411f8..04e4bd6bd 100644 --- a/README.rst +++ b/README.rst @@ -61,9 +61,8 @@ psutil currently supports the following platforms: - **Sun Solaris** - **AIX** -...both **32-bit** and **64-bit** architectures, with Python -versions from **2.6 to 3.6**. -`PyPy `__ is also known to work. +...both **32-bit** and **64-bit** architectures, with Python versions **2.6, +2.7, and 3.4+**. `PyPy `__ is also known to work. ==================== Example applications From bb92699432c6479d314b8c6a4d0d65be9de75d29 Mon Sep 17 00:00:00 2001 From: Jon Dufresne Date: Thu, 8 Mar 2018 11:21:55 -0800 Subject: [PATCH 015/182] In tox, move flake8 to own target (#1235) Allows running static analysis (lint/flake8) separately from testing the library. Reduces the redundant times static analysis runs. Previously ran after every test environment, but only needs to run once, requiring less tests resources. For lint, use "skip_install = True" as installing psutil isn't required to do static analysis. For details on this configuration, see: https://tox.readthedocs.io/en/latest/config.html#confval-skip_install=BOOL > skip_install=BOOL > > Do not install the current package. This can be used when you need the > virtualenv management but do not want to install the current package > into that environment. --- tox.ini | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tox.ini b/tox.ini index ab8174ca3..61d63eee7 100644 --- a/tox.ini +++ b/tox.ini @@ -5,11 +5,10 @@ # directory. [tox] -envlist = py26, py27, py34, py35, py36 +envlist = py26, py27, py34, py35, py36, lint [testenv] deps = - flake8 py26: ipaddress py26: mock==1.0.1 py26: unittest2 @@ -19,10 +18,11 @@ deps = setenv = TOX = 1 -commands = - python psutil/tests/__main__.py - git ls-files | grep \\.py$ | xargs flake8 +commands = python psutil/tests/__main__.py -# suppress "WARNING: 'git' command found but not installed in testenv -whitelist_externals = git usedevelop = True + +[testenv:lint] +deps = flake8 +commands = flake8 +skip_install = True From b409472c271c0f803cfa8ecc83033e508f2ae429 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 8 Mar 2018 20:51:37 +0100 Subject: [PATCH 016/182] test environ: clean returned dict --- psutil/tests/test_process.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index a629cae52..33557b051 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -1410,7 +1410,9 @@ def clean_dict(d): d.pop("VERSIONER_PYTHON_PREFER_32_BIT", None) d.pop("VERSIONER_PYTHON_VERSION", None) return dict( - [(k.rstrip("\r\n"), v.rstrip("\r\n")) for k, v in d.items()]) + [(k.replace("\r", "").replace("\n", ""), + v.replace("\r", "").replace("\n", "")) + for k, v in d.items()]) self.maxDiff = None p = psutil.Process() From b578d2febfd35f80fcfa1ce1bdf18d44d21b1581 Mon Sep 17 00:00:00 2001 From: Jon Dufresne Date: Fri, 9 Mar 2018 19:51:10 -0800 Subject: [PATCH 017/182] Move tests out of package to top level directory (#1232) Cleanly separates tests from the package itself. Prevents the tests being installed to site-packages. Tests are still distributed with the source distribution by MANIFEST.in. Avoids installing tests in production environments, leading to less total code in the environment. --- .ci/travis/run.sh | 6 +- .coveragerc | 2 +- DEVGUIDE.rst | 15 ++- MANIFEST.in | 34 +++---- Makefile | 22 ++--- appveyor.yml | 16 ++-- docs/index.rst | 7 +- scripts/internal/winmake.py | 14 +-- setup.py | 4 +- {psutil/tests => tests}/README.rst | 9 +- {psutil/tests => tests}/__init__.py | 4 +- {psutil/tests => tests}/__main__.py | 12 +-- {psutil/tests => tests}/test_aix.py | 12 +-- {psutil/tests => tests}/test_bsd.py | 18 ++-- {psutil/tests => tests}/test_connections.py | 40 ++++---- {psutil/tests => tests}/test_contracts.py | 32 +++---- {psutil/tests => tests}/test_linux.py | 40 ++++---- {psutil/tests => tests}/test_memory_leaks.py | 40 ++++---- {psutil/tests => tests}/test_misc.py | 96 ++++++++++---------- {psutil/tests => tests}/test_osx.py | 18 ++-- {psutil/tests => tests}/test_posix.py | 28 +++--- {psutil/tests => tests}/test_process.py | 68 +++++++------- {psutil/tests => tests}/test_sunos.py | 6 +- {psutil/tests => tests}/test_system.py | 40 ++++---- {psutil/tests => tests}/test_unicode.py | 44 ++++----- {psutil/tests => tests}/test_windows.py | 18 ++-- tox.ini | 2 +- 27 files changed, 316 insertions(+), 331 deletions(-) rename {psutil/tests => tests}/README.rst (62%) rename {psutil/tests => tests}/__init__.py (99%) rename {psutil/tests => tests}/__main__.py (90%) rename {psutil/tests => tests}/test_aix.py (93%) rename {psutil/tests => tests}/test_bsd.py (98%) rename {psutil/tests => tests}/test_connections.py (96%) rename {psutil/tests => tests}/test_contracts.py (97%) rename {psutil/tests => tests}/test_linux.py (99%) rename {psutil/tests => tests}/test_memory_leaks.py (95%) rename {psutil/tests => tests}/test_misc.py (94%) rename {psutil/tests => tests}/test_osx.py (96%) rename {psutil/tests => tests}/test_posix.py (96%) rename {psutil/tests => tests}/test_process.py (97%) rename {psutil/tests => tests}/test_sunos.py (91%) rename {psutil/tests => tests}/test_system.py (97%) rename {psutil/tests => tests}/test_unicode.py (93%) rename {psutil/tests => tests}/test_windows.py (98%) diff --git a/.ci/travis/run.sh b/.ci/travis/run.sh index 1501387a5..6660bf533 100755 --- a/.ci/travis/run.sh +++ b/.ci/travis/run.sh @@ -20,14 +20,14 @@ python setup.py develop # run tests (with coverage) if [[ $PYVER == '2.7' ]] && [[ "$(uname -s)" != 'Darwin' ]]; then - PSUTIL_TESTING=1 python -Wa -m coverage run psutil/tests/__main__.py + PSUTIL_TESTING=1 python -Wa -m coverage run tests/__main__.py else - PSUTIL_TESTING=1 python -Wa psutil/tests/__main__.py + PSUTIL_TESTING=1 python -Wa tests/__main__.py fi if [ "$PYVER" == "2.7" ] || [ "$PYVER" == "3.6" ]; then # run mem leaks test - PSUTIL_TESTING=1 python -Wa psutil/tests/test_memory_leaks.py + PSUTIL_TESTING=1 python -Wa tests/test_memory_leaks.py # run linter (on Linux only) if [[ "$(uname -s)" != 'Darwin' ]]; then python -m flake8 diff --git a/.coveragerc b/.coveragerc index 7d3f185f5..28316cfb8 100644 --- a/.coveragerc +++ b/.coveragerc @@ -4,7 +4,7 @@ include = *psutil* omit = psutil/_compat.py - psutil/tests/* + tests/* setup.py exclude_lines = enum.IntEnum diff --git a/DEVGUIDE.rst b/DEVGUIDE.rst index 2d48ced27..df4c9cbbf 100644 --- a/DEVGUIDE.rst +++ b/DEVGUIDE.rst @@ -99,8 +99,8 @@ Usually the files involved when adding a new functionality are: psutil/_ps{platform}.py # python platform wrapper psutil/_psutil_{platform}.c # C platform extension psutil/_psutil_{platform}.h # C header file - psutil/tests/test_process|system.py # main test suite - psutil/tests/test_{platform}.py # platform specific test suite + tests/test_process|system.py # main test suite + tests/test_{platform}.py # platform specific test suite Typical process occurring when adding a new functionality (API): @@ -109,12 +109,11 @@ Typical process occurring when adding a new functionality (API): (e.g. ``psutil/_pslinux.py``). - if the change requires C, write the C implementation in ``psutil/_psutil_{platform}.c`` (e.g. ``psutil/_psutil_linux.c``). -- write a generic test in ``psutil/tests/test_system.py`` or - ``psutil/tests/test_process.py``. -- if possible, write a platform specific test in - ``psutil/tests/test_{platform}.py`` (e.g. ``test_linux.py``). - This usually means testing the return value of the new feature against - a system CLI tool. +- write a generic test in ``tests/test_system.py`` or + ``tests/test_process.py``. +- if possible, write a platform specific test in ``tests/test_{platform}.py`` + (e.g. ``test_linux.py``). This usually means testing the return value of the + new feature against a system CLI tool. - update doc in ``doc/index.py``. - update ``HISTORY.rst``. - update ``README.rst`` (if necessary). diff --git a/MANIFEST.in b/MANIFEST.in index 7a92a4e5a..c422be868 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -79,23 +79,6 @@ include psutil/arch/windows/security.c include psutil/arch/windows/security.h include psutil/arch/windows/services.c include psutil/arch/windows/services.h -include psutil/tests/README.rst -include psutil/tests/__init__.py -include psutil/tests/__main__.py -include psutil/tests/test_aix.py -include psutil/tests/test_bsd.py -include psutil/tests/test_connections.py -include psutil/tests/test_contracts.py -include psutil/tests/test_linux.py -include psutil/tests/test_memory_leaks.py -include psutil/tests/test_misc.py -include psutil/tests/test_osx.py -include psutil/tests/test_posix.py -include psutil/tests/test_process.py -include psutil/tests/test_sunos.py -include psutil/tests/test_system.py -include psutil/tests/test_unicode.py -include psutil/tests/test_windows.py include scripts/battery.py include scripts/cpu_distribution.py include scripts/disk_usage.py @@ -128,4 +111,21 @@ include scripts/top.py include scripts/who.py include scripts/winservices.py include setup.py +include tests/README.rst +include tests/__init__.py +include tests/__main__.py +include tests/test_aix.py +include tests/test_bsd.py +include tests/test_connections.py +include tests/test_contracts.py +include tests/test_linux.py +include tests/test_memory_leaks.py +include tests/test_misc.py +include tests/test_osx.py +include tests/test_posix.py +include tests/test_process.py +include tests/test_sunos.py +include tests/test_system.py +include tests/test_unicode.py +include tests/test_windows.py include tox.ini diff --git a/Makefile b/Makefile index 5081a4edd..99355233e 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ # You can set the variables below from the command line. PYTHON = python -TSCRIPT = psutil/tests/__main__.py +TSCRIPT = tests/__main__.py ARGS = # List of nice-to-have dev libs. DEPS = \ @@ -113,41 +113,41 @@ test: ## Run all tests. test-process: ## Run process-related API tests. ${MAKE} install - $(TEST_PREFIX) $(PYTHON) -m unittest -v psutil.tests.test_process + $(TEST_PREFIX) $(PYTHON) -m unittest -v tests.test_process test-system: ## Run system-related API tests. ${MAKE} install - $(TEST_PREFIX) $(PYTHON) -m unittest -v psutil.tests.test_system + $(TEST_PREFIX) $(PYTHON) -m unittest -v tests.test_system test-misc: ## Run miscellaneous tests. ${MAKE} install - $(TEST_PREFIX) $(PYTHON) psutil/tests/test_misc.py + $(TEST_PREFIX) $(PYTHON) tests/test_misc.py test-unicode: ## Test APIs dealing with strings. ${MAKE} install - $(TEST_PREFIX) $(PYTHON) psutil/tests/test_unicode.py + $(TEST_PREFIX) $(PYTHON) tests/test_unicode.py test-contracts: ## APIs sanity tests. ${MAKE} install - $(TEST_PREFIX) $(PYTHON) psutil/tests/test_contracts.py + $(TEST_PREFIX) $(PYTHON) tests/test_contracts.py test-connections: ## Test net_connections() and Process.connections(). ${MAKE} install - $(TEST_PREFIX) $(PYTHON) psutil/tests/test_connections.py + $(TEST_PREFIX) $(PYTHON) tests/test_connections.py test-posix: ## POSIX specific tests. ${MAKE} install - $(TEST_PREFIX) $(PYTHON) psutil/tests/test_posix.py + $(TEST_PREFIX) $(PYTHON) tests/test_posix.py test-platform: ## Run specific platform tests only. ${MAKE} install - $(TEST_PREFIX) $(PYTHON) psutil/tests/test_`$(PYTHON) -c 'import psutil; print([x.lower() for x in ("LINUX", "BSD", "OSX", "SUNOS", "WINDOWS", "AIX") if getattr(psutil, x)][0])'`.py + $(TEST_PREFIX) $(PYTHON) tests/test_`$(PYTHON) -c 'import psutil; print([x.lower() for x in ("LINUX", "BSD", "OSX", "SUNOS", "WINDOWS", "AIX") if getattr(psutil, x)][0])'`.py test-memleaks: ## Memory leak tests. ${MAKE} install - $(TEST_PREFIX) $(PYTHON) psutil/tests/test_memory_leaks.py + $(TEST_PREFIX) $(PYTHON) tests/test_memory_leaks.py -test-by-name: ## e.g. make test-by-name ARGS=psutil.tests.test_system.TestSystemAPIs +test-by-name: ## e.g. make test-by-name ARGS=tests.test_system.TestSystemAPIs ${MAKE} install @$(TEST_PREFIX) $(PYTHON) -m unittest -v $(ARGS) diff --git a/appveyor.yml b/appveyor.yml index 28f5f7f6f..2ceb63f79 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -83,7 +83,7 @@ build: off test_script: - "%WITH_COMPILER% %PYTHON%/python -V" - - "set PYTHONWARNINGS=all && set PSUTIL_TESTING=1 && set PSUTIL_DEBUG=1 && %WITH_COMPILER% %PYTHON%/python psutil/tests/__main__.py" + - "set PYTHONWARNINGS=all && set PSUTIL_TESTING=1 && set PSUTIL_DEBUG=1 && %WITH_COMPILER% %PYTHON%/python tests/__main__.py" after_test: - "%WITH_COMPILER% %PYTHON%/python setup.py bdist_wheel" @@ -112,12 +112,12 @@ only_commits: psutil/_psutil_windows.* psutil/_pswindows.py psutil/arch/windows/* - psutil/tests/__init__.py - psutil/tests/__main__.py - psutil/tests/test_memory_leaks.py - psutil/tests/test_misc.py - psutil/tests/test_process.py - psutil/tests/test_system.py - psutil/tests/test_windows.py + tests/__init__.py + tests/__main__.py + tests/test_memory_leaks.py + tests/test_misc.py + tests/test_process.py + tests/test_system.py + tests/test_windows.py scripts/* setup.py diff --git a/docs/index.rst b/docs/index.rst index 4f77b2e49..5ea03ab82 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -2605,12 +2605,7 @@ FAQs Running tests ============= -There are two ways of running tests. If psutil is already installed use:: - - $ python -m psutil.tests - -You can use this method as a quick way to make sure psutil fully works on your -platform. If you have a copy of the source code you can also use:: +First, obtain a copy of the source code. Then run tests using the command:: $ make test diff --git a/scripts/internal/winmake.py b/scripts/internal/winmake.py index 548f7a8ed..de58fbfca 100755 --- a/scripts/internal/winmake.py +++ b/scripts/internal/winmake.py @@ -355,7 +355,7 @@ def test_process(): """Run process tests""" install() test_setup() - sh("%s -m unittest -v psutil.tests.test_process" % PYTHON) + sh("%s -m unittest -v tests.test_process" % PYTHON) @cmd @@ -363,7 +363,7 @@ def test_system(): """Run system tests""" install() test_setup() - sh("%s -m unittest -v psutil.tests.test_system" % PYTHON) + sh("%s -m unittest -v tests.test_system" % PYTHON) @cmd @@ -371,7 +371,7 @@ def test_platform(): """Run windows only tests""" install() test_setup() - sh("%s -m unittest -v psutil.tests.test_windows" % PYTHON) + sh("%s -m unittest -v tests.test_windows" % PYTHON) @cmd @@ -379,7 +379,7 @@ def test_misc(): """Run misc tests""" install() test_setup() - sh("%s -m unittest -v psutil.tests.test_misc" % PYTHON) + sh("%s -m unittest -v tests.test_misc" % PYTHON) @cmd @@ -387,7 +387,7 @@ def test_unicode(): """Run unicode tests""" install() test_setup() - sh("%s -m unittest -v psutil.tests.test_unicode" % PYTHON) + sh("%s -m unittest -v tests.test_unicode" % PYTHON) @cmd @@ -395,7 +395,7 @@ def test_connections(): """Run connections tests""" install() test_setup() - sh("%s -m unittest -v psutil.tests.test_connections" % PYTHON) + sh("%s -m unittest -v tests.test_connections" % PYTHON) @cmd @@ -403,7 +403,7 @@ def test_contracts(): """Run contracts tests""" install() test_setup() - sh("%s -m unittest -v psutil.tests.test_contracts" % PYTHON) + sh("%s -m unittest -v tests.test_contracts" % PYTHON) @cmd diff --git a/setup.py b/setup.py index 61056f5f1..f03ab4570 100755 --- a/setup.py +++ b/setup.py @@ -279,7 +279,7 @@ def main(): url='https://github.com/giampaolo/psutil', platforms='Platform Independent', license='BSD', - packages=['psutil', 'psutil.tests'], + packages=['psutil'], ext_modules=extensions, # see: python setup.py register --list-classifiers classifiers=[ @@ -327,7 +327,7 @@ def main(): if setuptools is not None: kwargs.update( python_requires=">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*", - test_suite="psutil.tests.get_suite", + test_suite="tests.get_suite", tests_require=[ 'ipaddress; python_version < "3.3"', 'mock; python_version < "3.3"', diff --git a/psutil/tests/README.rst b/tests/README.rst similarity index 62% rename from psutil/tests/README.rst rename to tests/README.rst index 515abf772..a02b88134 100644 --- a/psutil/tests/README.rst +++ b/tests/README.rst @@ -1,14 +1,7 @@ Instructions for running tests ============================== -* There are two ways of running tests. As a "user", if psutil is already - installed and you just want to test it works:: - - python -m psutil.tests --install-deps # install test deps - python -m psutil.tests - - As a "developer", if you have a copy of the source code and you whish to hack - on psutil:: +* To run tests:: make setup-dev-env # install test deps (+ other things) make test diff --git a/psutil/tests/__init__.py b/tests/__init__.py similarity index 99% rename from psutil/tests/__init__.py rename to tests/__init__.py index 499240fb2..f59cd846e 100644 --- a/psutil/tests/__init__.py +++ b/tests/__init__.py @@ -142,7 +142,7 @@ # --- paths -ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..')) +ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) SCRIPTS_DIR = os.path.join(ROOT_DIR, 'scripts') HERE = os.path.abspath(os.path.dirname(__file__)) @@ -788,7 +788,7 @@ def get_suite(): suite = unittest.TestSuite() for tm in testmods: # ...so that the full test paths are printed on screen - tm = "psutil.tests.%s" % tm + tm = "tests.%s" % tm suite.addTest(unittest.defaultTestLoader.loadTestsFromName(tm)) return suite diff --git a/psutil/tests/__main__.py b/tests/__main__.py similarity index 90% rename from psutil/tests/__main__.py rename to tests/__main__.py index 62fe07428..97690afeb 100755 --- a/psutil/tests/__main__.py +++ b/tests/__main__.py @@ -5,9 +5,7 @@ # found in the LICENSE file. """ -Run unit tests. This is invoked by: - -$ python -m psutil.tests +Run unit tests. """ import contextlib @@ -21,8 +19,8 @@ except ImportError: from urllib2 import urlopen -from psutil.tests import PYTHON_EXE -from psutil.tests import run_suite +from tests import PYTHON_EXE +from tests import run_suite HERE = os.path.abspath(os.path.dirname(__file__)) @@ -71,7 +69,7 @@ def install_test_deps(deps=None): def main(): - usage = "%s -m psutil.tests [opts]" % PYTHON_EXE + usage = "%s -m tests [opts]" % PYTHON_EXE parser = optparse.OptionParser(usage=usage, description="run unit tests") parser.add_option("-i", "--install-deps", action="store_true", default=False, @@ -86,7 +84,7 @@ def main(): try: __import__(dep.split("==")[0]) except ImportError: - sys.exit("%r lib is not installed; run %s -m psutil.tests " + sys.exit("%r lib is not installed; run %s -m tests " "--install-deps" % (dep, PYTHON_EXE)) run_suite() diff --git a/psutil/tests/test_aix.py b/tests/test_aix.py similarity index 93% rename from psutil/tests/test_aix.py rename to tests/test_aix.py index 7a8a4c334..8370c3830 100755 --- a/psutil/tests/test_aix.py +++ b/tests/test_aix.py @@ -11,9 +11,9 @@ import re from psutil import AIX -from psutil.tests import run_test_module_by_name -from psutil.tests import sh -from psutil.tests import unittest +from tests import run_test_module_by_name +from tests import sh +from tests import unittest import psutil @@ -38,9 +38,9 @@ def test_virtual_memory(self): psutil_result = psutil.virtual_memory() - # MEMORY_TOLERANCE from psutil.tests is not enough. For some reason - # we're seeing differences of ~1.2 MB. 2 MB is still a good tolerance - # when compared to GBs. + # MEMORY_TOLERANCE from tests is not enough. For some reason we're + # seeing differences of ~1.2 MB. 2 MB is still a good tolerance when + # compared to GBs. MEMORY_TOLERANCE = 2 * KB * KB # 2 MB self.assertEqual(psutil_result.total, total) self.assertAlmostEqual( diff --git a/psutil/tests/test_bsd.py b/tests/test_bsd.py similarity index 98% rename from psutil/tests/test_bsd.py rename to tests/test_bsd.py index d3868ada1..cc9c058d3 100755 --- a/psutil/tests/test_bsd.py +++ b/tests/test_bsd.py @@ -20,15 +20,15 @@ from psutil import FREEBSD from psutil import NETBSD from psutil import OPENBSD -from psutil.tests import get_test_subprocess -from psutil.tests import HAS_BATTERY -from psutil.tests import MEMORY_TOLERANCE -from psutil.tests import reap_children -from psutil.tests import retry_before_failing -from psutil.tests import run_test_module_by_name -from psutil.tests import sh -from psutil.tests import unittest -from psutil.tests import which +from tests import get_test_subprocess +from tests import HAS_BATTERY +from tests import MEMORY_TOLERANCE +from tests import reap_children +from tests import retry_before_failing +from tests import run_test_module_by_name +from tests import sh +from tests import unittest +from tests import which if BSD: diff --git a/psutil/tests/test_connections.py b/tests/test_connections.py similarity index 96% rename from psutil/tests/test_connections.py rename to tests/test_connections.py index 176e26648..c61e0c2c7 100755 --- a/psutil/tests/test_connections.py +++ b/tests/test_connections.py @@ -26,25 +26,25 @@ from psutil import WINDOWS from psutil._common import supports_ipv6 from psutil._compat import PY3 -from psutil.tests import AF_UNIX -from psutil.tests import bind_socket -from psutil.tests import bind_unix_socket -from psutil.tests import check_connection_ntuple -from psutil.tests import create_sockets -from psutil.tests import get_free_port -from psutil.tests import HAS_CONNECTIONS_UNIX -from psutil.tests import pyrun -from psutil.tests import reap_children -from psutil.tests import run_test_module_by_name -from psutil.tests import safe_rmpath -from psutil.tests import skip_on_access_denied -from psutil.tests import tcp_socketpair -from psutil.tests import TESTFN -from psutil.tests import TRAVIS -from psutil.tests import unittest -from psutil.tests import unix_socket_path -from psutil.tests import unix_socketpair -from psutil.tests import wait_for_file +from tests import AF_UNIX +from tests import bind_socket +from tests import bind_unix_socket +from tests import check_connection_ntuple +from tests import create_sockets +from tests import get_free_port +from tests import HAS_CONNECTIONS_UNIX +from tests import pyrun +from tests import reap_children +from tests import run_test_module_by_name +from tests import safe_rmpath +from tests import skip_on_access_denied +from tests import tcp_socketpair +from tests import TESTFN +from tests import TRAVIS +from tests import unittest +from tests import unix_socket_path +from tests import unix_socketpair +from tests import wait_for_file thisproc = psutil.Process() @@ -471,7 +471,7 @@ def test_multi_sockets_procs(self): fname = os.path.realpath(TESTFN) + str(i) src = textwrap.dedent("""\ import time, os - from psutil.tests import create_sockets + from tests import create_sockets with create_sockets(): with open('%s', 'w') as f: f.write(str(os.getpid())) diff --git a/psutil/tests/test_contracts.py b/tests/test_contracts.py similarity index 97% rename from psutil/tests/test_contracts.py rename to tests/test_contracts.py index 0dfb3e2a8..f07c2624e 100755 --- a/psutil/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -28,22 +28,22 @@ from psutil import SUNOS from psutil import WINDOWS from psutil._compat import long -from psutil.tests import bind_unix_socket -from psutil.tests import check_connection_ntuple -from psutil.tests import get_kernel_version -from psutil.tests import HAS_CONNECTIONS_UNIX -from psutil.tests import HAS_RLIMIT -from psutil.tests import HAS_SENSORS_FANS -from psutil.tests import HAS_SENSORS_TEMPERATURES -from psutil.tests import is_namedtuple -from psutil.tests import run_test_module_by_name -from psutil.tests import safe_rmpath -from psutil.tests import skip_on_access_denied -from psutil.tests import TESTFN -from psutil.tests import unittest -from psutil.tests import unix_socket_path -from psutil.tests import VALID_PROC_STATUSES -from psutil.tests import warn +from tests import bind_unix_socket +from tests import check_connection_ntuple +from tests import get_kernel_version +from tests import HAS_CONNECTIONS_UNIX +from tests import HAS_RLIMIT +from tests import HAS_SENSORS_FANS +from tests import HAS_SENSORS_TEMPERATURES +from tests import is_namedtuple +from tests import run_test_module_by_name +from tests import safe_rmpath +from tests import skip_on_access_denied +from tests import TESTFN +from tests import unittest +from tests import unix_socket_path +from tests import VALID_PROC_STATUSES +from tests import warn import psutil diff --git a/psutil/tests/test_linux.py b/tests/test_linux.py similarity index 99% rename from psutil/tests/test_linux.py rename to tests/test_linux.py index 6ba17b254..7cc5a01ac 100755 --- a/psutil/tests/test_linux.py +++ b/tests/test_linux.py @@ -27,26 +27,26 @@ from psutil import LINUX from psutil._compat import PY3 from psutil._compat import u -from psutil.tests import call_until -from psutil.tests import HAS_BATTERY -from psutil.tests import HAS_CPU_FREQ -from psutil.tests import HAS_RLIMIT -from psutil.tests import MEMORY_TOLERANCE -from psutil.tests import mock -from psutil.tests import PYPY -from psutil.tests import pyrun -from psutil.tests import reap_children -from psutil.tests import reload_module -from psutil.tests import retry_before_failing -from psutil.tests import run_test_module_by_name -from psutil.tests import safe_rmpath -from psutil.tests import sh -from psutil.tests import skip_on_not_implemented -from psutil.tests import TESTFN -from psutil.tests import ThreadTask -from psutil.tests import TRAVIS -from psutil.tests import unittest -from psutil.tests import which +from tests import call_until +from tests import HAS_BATTERY +from tests import HAS_CPU_FREQ +from tests import HAS_RLIMIT +from tests import MEMORY_TOLERANCE +from tests import mock +from tests import PYPY +from tests import pyrun +from tests import reap_children +from tests import reload_module +from tests import retry_before_failing +from tests import run_test_module_by_name +from tests import safe_rmpath +from tests import sh +from tests import skip_on_not_implemented +from tests import TESTFN +from tests import ThreadTask +from tests import TRAVIS +from tests import unittest +from tests import which HERE = os.path.abspath(os.path.dirname(__file__)) diff --git a/psutil/tests/test_memory_leaks.py b/tests/test_memory_leaks.py similarity index 95% rename from psutil/tests/test_memory_leaks.py rename to tests/test_memory_leaks.py index 680fe7803..53723c7bd 100755 --- a/psutil/tests/test_memory_leaks.py +++ b/tests/test_memory_leaks.py @@ -31,26 +31,26 @@ from psutil import SUNOS from psutil import WINDOWS from psutil._compat import xrange -from psutil.tests import create_sockets -from psutil.tests import get_test_subprocess -from psutil.tests import HAS_CPU_AFFINITY -from psutil.tests import HAS_CPU_FREQ -from psutil.tests import HAS_ENVIRON -from psutil.tests import HAS_IONICE -from psutil.tests import HAS_MEMORY_MAPS -from psutil.tests import HAS_PROC_CPU_NUM -from psutil.tests import HAS_PROC_IO_COUNTERS -from psutil.tests import HAS_RLIMIT -from psutil.tests import HAS_SENSORS_BATTERY -from psutil.tests import HAS_SENSORS_FANS -from psutil.tests import HAS_SENSORS_TEMPERATURES -from psutil.tests import reap_children -from psutil.tests import run_test_module_by_name -from psutil.tests import safe_rmpath -from psutil.tests import skip_on_access_denied -from psutil.tests import TESTFN -from psutil.tests import TRAVIS -from psutil.tests import unittest +from tests import create_sockets +from tests import get_test_subprocess +from tests import HAS_CPU_AFFINITY +from tests import HAS_CPU_FREQ +from tests import HAS_ENVIRON +from tests import HAS_IONICE +from tests import HAS_MEMORY_MAPS +from tests import HAS_PROC_CPU_NUM +from tests import HAS_PROC_IO_COUNTERS +from tests import HAS_RLIMIT +from tests import HAS_SENSORS_BATTERY +from tests import HAS_SENSORS_FANS +from tests import HAS_SENSORS_TEMPERATURES +from tests import reap_children +from tests import run_test_module_by_name +from tests import safe_rmpath +from tests import skip_on_access_denied +from tests import TESTFN +from tests import TRAVIS +from tests import unittest LOOPS = 1000 diff --git a/psutil/tests/test_misc.py b/tests/test_misc.py similarity index 94% rename from psutil/tests/test_misc.py rename to tests/test_misc.py index 1d9067e77..3dbd49d17 100755 --- a/psutil/tests/test_misc.py +++ b/tests/test_misc.py @@ -27,48 +27,48 @@ from psutil._common import supports_ipv6 from psutil._common import wrap_numbers from psutil._compat import PY3 -from psutil.tests import APPVEYOR -from psutil.tests import bind_socket -from psutil.tests import bind_unix_socket -from psutil.tests import call_until -from psutil.tests import chdir -from psutil.tests import create_proc_children_pair -from psutil.tests import create_sockets -from psutil.tests import create_zombie_proc -from psutil.tests import DEVNULL -from psutil.tests import get_free_port -from psutil.tests import get_test_subprocess -from psutil.tests import HAS_BATTERY -from psutil.tests import HAS_CONNECTIONS_UNIX -from psutil.tests import HAS_MEMORY_FULL_INFO -from psutil.tests import HAS_MEMORY_MAPS -from psutil.tests import HAS_SENSORS_BATTERY -from psutil.tests import HAS_SENSORS_FANS -from psutil.tests import HAS_SENSORS_TEMPERATURES -from psutil.tests import import_module_by_path -from psutil.tests import is_namedtuple -from psutil.tests import mock -from psutil.tests import PYTHON_EXE -from psutil.tests import reap_children -from psutil.tests import reload_module -from psutil.tests import retry -from psutil.tests import ROOT_DIR -from psutil.tests import run_test_module_by_name -from psutil.tests import safe_mkdir -from psutil.tests import safe_rmpath -from psutil.tests import SCRIPTS_DIR -from psutil.tests import sh -from psutil.tests import tcp_socketpair -from psutil.tests import TESTFN -from psutil.tests import TOX -from psutil.tests import TRAVIS -from psutil.tests import unittest -from psutil.tests import unix_socket_path -from psutil.tests import unix_socketpair -from psutil.tests import wait_for_file -from psutil.tests import wait_for_pid +from tests import APPVEYOR +from tests import bind_socket +from tests import bind_unix_socket +from tests import call_until +from tests import chdir +from tests import create_proc_children_pair +from tests import create_sockets +from tests import create_zombie_proc +from tests import DEVNULL +from tests import get_free_port +from tests import get_test_subprocess +from tests import HAS_BATTERY +from tests import HAS_CONNECTIONS_UNIX +from tests import HAS_MEMORY_FULL_INFO +from tests import HAS_MEMORY_MAPS +from tests import HAS_SENSORS_BATTERY +from tests import HAS_SENSORS_FANS +from tests import HAS_SENSORS_TEMPERATURES +from tests import import_module_by_path +from tests import is_namedtuple +from tests import mock +from tests import PYTHON_EXE +from tests import reap_children +from tests import reload_module +from tests import retry +from tests import ROOT_DIR +from tests import run_test_module_by_name +from tests import safe_mkdir +from tests import safe_rmpath +from tests import SCRIPTS_DIR +from tests import sh +from tests import tcp_socketpair +from tests import TESTFN +from tests import TOX +from tests import TRAVIS +from tests import unittest +from tests import unix_socket_path +from tests import unix_socketpair +from tests import wait_for_file +from tests import wait_for_pid import psutil -import psutil.tests +import tests # =================================================================== @@ -859,7 +859,7 @@ def tearDown(self): def test_wait_for_pid(self): wait_for_pid(os.getpid()) nopid = max(psutil.pids()) + 99999 - with mock.patch('psutil.tests.retry.__iter__', return_value=iter([0])): + with mock.patch('tests.retry.__iter__', return_value=iter([0])): self.assertRaises(psutil.NoSuchProcess, wait_for_pid, nopid) def test_wait_for_file(self): @@ -875,7 +875,7 @@ def test_wait_for_file_empty(self): assert not os.path.exists(TESTFN) def test_wait_for_file_no_file(self): - with mock.patch('psutil.tests.retry.__iter__', return_value=iter([0])): + with mock.patch('tests.retry.__iter__', return_value=iter([0])): self.assertRaises(IOError, wait_for_file, TESTFN) def test_wait_for_file_no_delete(self): @@ -914,7 +914,7 @@ def test_safe_rmpath(self): safe_rmpath(TESTFN) assert not os.path.exists(TESTFN) # test other exceptions are raised - with mock.patch('psutil.tests.os.stat', + with mock.patch('tests.os.stat', side_effect=OSError(errno.EINVAL, "")) as m: with self.assertRaises(OSError): safe_rmpath(TESTFN) @@ -936,8 +936,8 @@ def test_reap_children(self): assert p.is_running() reap_children() assert not p.is_running() - assert not psutil.tests._pids_started - assert not psutil.tests._subprocesses_started + assert not tests._pids_started + assert not tests._subprocesses_started def test_create_proc_children_pair(self): p1, p2 = create_proc_children_pair() @@ -955,8 +955,8 @@ def test_create_proc_children_pair(self): reap_children() assert not p1.is_running() assert not p2.is_running() - assert not psutil.tests._pids_started - assert not psutil.tests._subprocesses_started + assert not tests._pids_started + assert not tests._subprocesses_started @unittest.skipIf(not POSIX, "POSIX only") def test_create_zombie_proc(self): diff --git a/psutil/tests/test_osx.py b/tests/test_osx.py similarity index 96% rename from psutil/tests/test_osx.py rename to tests/test_osx.py index bcb2ba4e1..4e8b2e470 100755 --- a/psutil/tests/test_osx.py +++ b/tests/test_osx.py @@ -12,15 +12,15 @@ import psutil from psutil import OSX -from psutil.tests import create_zombie_proc -from psutil.tests import get_test_subprocess -from psutil.tests import HAS_BATTERY -from psutil.tests import MEMORY_TOLERANCE -from psutil.tests import reap_children -from psutil.tests import retry_before_failing -from psutil.tests import run_test_module_by_name -from psutil.tests import sh -from psutil.tests import unittest +from tests import create_zombie_proc +from tests import get_test_subprocess +from tests import HAS_BATTERY +from tests import MEMORY_TOLERANCE +from tests import reap_children +from tests import retry_before_failing +from tests import run_test_module_by_name +from tests import sh +from tests import unittest PAGESIZE = os.sysconf("SC_PAGE_SIZE") if OSX else None diff --git a/psutil/tests/test_posix.py b/tests/test_posix.py similarity index 96% rename from psutil/tests/test_posix.py rename to tests/test_posix.py index e9a6f5f63..382ebade6 100755 --- a/psutil/tests/test_posix.py +++ b/tests/test_posix.py @@ -24,20 +24,20 @@ from psutil import POSIX from psutil import SUNOS from psutil._compat import PY3 -from psutil.tests import APPVEYOR -from psutil.tests import get_kernel_version -from psutil.tests import get_test_subprocess -from psutil.tests import mock -from psutil.tests import PYTHON_EXE -from psutil.tests import reap_children -from psutil.tests import retry_before_failing -from psutil.tests import run_test_module_by_name -from psutil.tests import sh -from psutil.tests import skip_on_access_denied -from psutil.tests import TRAVIS -from psutil.tests import unittest -from psutil.tests import wait_for_pid -from psutil.tests import which +from tests import APPVEYOR +from tests import get_kernel_version +from tests import get_test_subprocess +from tests import mock +from tests import PYTHON_EXE +from tests import reap_children +from tests import retry_before_failing +from tests import run_test_module_by_name +from tests import sh +from tests import skip_on_access_denied +from tests import TRAVIS +from tests import unittest +from tests import wait_for_pid +from tests import which def ps(cmd): diff --git a/psutil/tests/test_process.py b/tests/test_process.py similarity index 97% rename from psutil/tests/test_process.py rename to tests/test_process.py index 33557b051..1c0d6a8cd 100755 --- a/psutil/tests/test_process.py +++ b/tests/test_process.py @@ -33,40 +33,40 @@ from psutil import WINDOWS from psutil._compat import long from psutil._compat import PY3 -from psutil.tests import APPVEYOR -from psutil.tests import call_until -from psutil.tests import copyload_shared_lib -from psutil.tests import create_exe -from psutil.tests import create_proc_children_pair -from psutil.tests import create_zombie_proc -from psutil.tests import enum -from psutil.tests import get_test_subprocess -from psutil.tests import get_winver -from psutil.tests import HAS_CPU_AFFINITY -from psutil.tests import HAS_ENVIRON -from psutil.tests import HAS_IONICE -from psutil.tests import HAS_MEMORY_MAPS -from psutil.tests import HAS_PROC_CPU_NUM -from psutil.tests import HAS_PROC_IO_COUNTERS -from psutil.tests import HAS_RLIMIT -from psutil.tests import HAS_THREADS -from psutil.tests import mock -from psutil.tests import PYPY -from psutil.tests import PYTHON_EXE -from psutil.tests import reap_children -from psutil.tests import retry_before_failing -from psutil.tests import run_test_module_by_name -from psutil.tests import safe_rmpath -from psutil.tests import sh -from psutil.tests import skip_on_access_denied -from psutil.tests import skip_on_not_implemented -from psutil.tests import TESTFILE_PREFIX -from psutil.tests import TESTFN -from psutil.tests import ThreadTask -from psutil.tests import TRAVIS -from psutil.tests import unittest -from psutil.tests import wait_for_pid -from psutil.tests import WIN_VISTA +from tests import APPVEYOR +from tests import call_until +from tests import copyload_shared_lib +from tests import create_exe +from tests import create_proc_children_pair +from tests import create_zombie_proc +from tests import enum +from tests import get_test_subprocess +from tests import get_winver +from tests import HAS_CPU_AFFINITY +from tests import HAS_ENVIRON +from tests import HAS_IONICE +from tests import HAS_MEMORY_MAPS +from tests import HAS_PROC_CPU_NUM +from tests import HAS_PROC_IO_COUNTERS +from tests import HAS_RLIMIT +from tests import HAS_THREADS +from tests import mock +from tests import PYPY +from tests import PYTHON_EXE +from tests import reap_children +from tests import retry_before_failing +from tests import run_test_module_by_name +from tests import safe_rmpath +from tests import sh +from tests import skip_on_access_denied +from tests import skip_on_not_implemented +from tests import TESTFILE_PREFIX +from tests import TESTFN +from tests import ThreadTask +from tests import TRAVIS +from tests import unittest +from tests import wait_for_pid +from tests import WIN_VISTA # =================================================================== diff --git a/psutil/tests/test_sunos.py b/tests/test_sunos.py similarity index 91% rename from psutil/tests/test_sunos.py rename to tests/test_sunos.py index ea9afcde0..e266fb64e 100755 --- a/psutil/tests/test_sunos.py +++ b/tests/test_sunos.py @@ -10,9 +10,9 @@ import psutil from psutil import SUNOS -from psutil.tests import run_test_module_by_name -from psutil.tests import sh -from psutil.tests import unittest +from tests import run_test_module_by_name +from tests import sh +from tests import unittest @unittest.skipIf(not SUNOS, "SUNOS only") diff --git a/psutil/tests/test_system.py b/tests/test_system.py similarity index 97% rename from psutil/tests/test_system.py rename to tests/test_system.py index 20b132a91..6eb891d4f 100755 --- a/psutil/tests/test_system.py +++ b/tests/test_system.py @@ -30,26 +30,26 @@ from psutil import SUNOS from psutil import WINDOWS from psutil._compat import long -from psutil.tests import APPVEYOR -from psutil.tests import ASCII_FS -from psutil.tests import check_net_address -from psutil.tests import DEVNULL -from psutil.tests import enum -from psutil.tests import get_test_subprocess -from psutil.tests import HAS_BATTERY -from psutil.tests import HAS_CPU_FREQ -from psutil.tests import HAS_SENSORS_BATTERY -from psutil.tests import HAS_SENSORS_FANS -from psutil.tests import HAS_SENSORS_TEMPERATURES -from psutil.tests import mock -from psutil.tests import reap_children -from psutil.tests import retry_before_failing -from psutil.tests import run_test_module_by_name -from psutil.tests import safe_rmpath -from psutil.tests import TESTFN -from psutil.tests import TESTFN_UNICODE -from psutil.tests import TRAVIS -from psutil.tests import unittest +from tests import APPVEYOR +from tests import ASCII_FS +from tests import check_net_address +from tests import DEVNULL +from tests import enum +from tests import get_test_subprocess +from tests import HAS_BATTERY +from tests import HAS_CPU_FREQ +from tests import HAS_SENSORS_BATTERY +from tests import HAS_SENSORS_FANS +from tests import HAS_SENSORS_TEMPERATURES +from tests import mock +from tests import reap_children +from tests import retry_before_failing +from tests import run_test_module_by_name +from tests import safe_rmpath +from tests import TESTFN +from tests import TESTFN_UNICODE +from tests import TRAVIS +from tests import unittest # =================================================================== diff --git a/psutil/tests/test_unicode.py b/tests/test_unicode.py similarity index 93% rename from psutil/tests/test_unicode.py rename to tests/test_unicode.py index 6383c9bec..ea1a6900a 100755 --- a/psutil/tests/test_unicode.py +++ b/tests/test_unicode.py @@ -64,28 +64,28 @@ from psutil import WINDOWS from psutil._compat import PY3 from psutil._compat import u -from psutil.tests import APPVEYOR -from psutil.tests import ASCII_FS -from psutil.tests import bind_unix_socket -from psutil.tests import chdir -from psutil.tests import copyload_shared_lib -from psutil.tests import create_exe -from psutil.tests import get_test_subprocess -from psutil.tests import HAS_CONNECTIONS_UNIX -from psutil.tests import HAS_ENVIRON -from psutil.tests import HAS_MEMORY_MAPS -from psutil.tests import mock -from psutil.tests import reap_children -from psutil.tests import run_test_module_by_name -from psutil.tests import safe_mkdir -from psutil.tests import safe_rmpath as _safe_rmpath -from psutil.tests import skip_on_access_denied -from psutil.tests import TESTFILE_PREFIX -from psutil.tests import TESTFN -from psutil.tests import TESTFN_UNICODE -from psutil.tests import TRAVIS -from psutil.tests import unittest -from psutil.tests import unix_socket_path +from tests import APPVEYOR +from tests import ASCII_FS +from tests import bind_unix_socket +from tests import chdir +from tests import copyload_shared_lib +from tests import create_exe +from tests import get_test_subprocess +from tests import HAS_CONNECTIONS_UNIX +from tests import HAS_ENVIRON +from tests import HAS_MEMORY_MAPS +from tests import mock +from tests import reap_children +from tests import run_test_module_by_name +from tests import safe_mkdir +from tests import safe_rmpath as _safe_rmpath +from tests import skip_on_access_denied +from tests import TESTFILE_PREFIX +from tests import TESTFN +from tests import TESTFN_UNICODE +from tests import TRAVIS +from tests import unittest +from tests import unix_socket_path import psutil diff --git a/psutil/tests/test_windows.py b/tests/test_windows.py similarity index 98% rename from psutil/tests/test_windows.py rename to tests/test_windows.py index 32c46f67a..236f2aefc 100755 --- a/psutil/tests/test_windows.py +++ b/tests/test_windows.py @@ -21,15 +21,15 @@ import psutil from psutil import WINDOWS -from psutil.tests import APPVEYOR -from psutil.tests import get_test_subprocess -from psutil.tests import HAS_BATTERY -from psutil.tests import mock -from psutil.tests import reap_children -from psutil.tests import retry_before_failing -from psutil.tests import run_test_module_by_name -from psutil.tests import sh -from psutil.tests import unittest +from tests import APPVEYOR +from tests import get_test_subprocess +from tests import HAS_BATTERY +from tests import mock +from tests import reap_children +from tests import retry_before_failing +from tests import run_test_module_by_name +from tests import sh +from tests import unittest with warnings.catch_warnings(): warnings.simplefilter("ignore") diff --git a/tox.ini b/tox.ini index 61d63eee7..7ec895af0 100644 --- a/tox.ini +++ b/tox.ini @@ -18,7 +18,7 @@ deps = setenv = TOX = 1 -commands = python psutil/tests/__main__.py +commands = python tests/__main__.py usedevelop = True From 2ffb0ecc5e41c5486056fcd00aca274d77858860 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 11 Mar 2018 09:31:35 -0700 Subject: [PATCH 018/182] Revert "Move tests out of package to top level directory (#1232)" (#1242) This reverts commit b578d2febfd35f80fcfa1ce1bdf18d44d21b1581. --- .ci/travis/run.sh | 6 +- .coveragerc | 2 +- DEVGUIDE.rst | 15 +-- MANIFEST.in | 34 +++---- Makefile | 22 ++--- appveyor.yml | 16 ++-- docs/index.rst | 7 +- {tests => psutil/tests}/README.rst | 9 +- {tests => psutil/tests}/__init__.py | 4 +- {tests => psutil/tests}/__main__.py | 12 ++- {tests => psutil/tests}/test_aix.py | 12 +-- {tests => psutil/tests}/test_bsd.py | 18 ++-- {tests => psutil/tests}/test_connections.py | 40 ++++---- {tests => psutil/tests}/test_contracts.py | 32 +++---- {tests => psutil/tests}/test_linux.py | 40 ++++---- {tests => psutil/tests}/test_memory_leaks.py | 40 ++++---- {tests => psutil/tests}/test_misc.py | 96 ++++++++++---------- {tests => psutil/tests}/test_osx.py | 18 ++-- {tests => psutil/tests}/test_posix.py | 28 +++--- {tests => psutil/tests}/test_process.py | 68 +++++++------- {tests => psutil/tests}/test_sunos.py | 6 +- {tests => psutil/tests}/test_system.py | 40 ++++---- {tests => psutil/tests}/test_unicode.py | 44 ++++----- {tests => psutil/tests}/test_windows.py | 18 ++-- scripts/internal/winmake.py | 14 +-- setup.py | 4 +- tox.ini | 2 +- 27 files changed, 331 insertions(+), 316 deletions(-) rename {tests => psutil/tests}/README.rst (62%) rename {tests => psutil/tests}/__init__.py (99%) rename {tests => psutil/tests}/__main__.py (90%) rename {tests => psutil/tests}/test_aix.py (93%) rename {tests => psutil/tests}/test_bsd.py (98%) rename {tests => psutil/tests}/test_connections.py (96%) rename {tests => psutil/tests}/test_contracts.py (97%) rename {tests => psutil/tests}/test_linux.py (99%) rename {tests => psutil/tests}/test_memory_leaks.py (95%) rename {tests => psutil/tests}/test_misc.py (94%) rename {tests => psutil/tests}/test_osx.py (96%) rename {tests => psutil/tests}/test_posix.py (96%) rename {tests => psutil/tests}/test_process.py (97%) rename {tests => psutil/tests}/test_sunos.py (91%) rename {tests => psutil/tests}/test_system.py (97%) rename {tests => psutil/tests}/test_unicode.py (93%) rename {tests => psutil/tests}/test_windows.py (98%) diff --git a/.ci/travis/run.sh b/.ci/travis/run.sh index 6660bf533..1501387a5 100755 --- a/.ci/travis/run.sh +++ b/.ci/travis/run.sh @@ -20,14 +20,14 @@ python setup.py develop # run tests (with coverage) if [[ $PYVER == '2.7' ]] && [[ "$(uname -s)" != 'Darwin' ]]; then - PSUTIL_TESTING=1 python -Wa -m coverage run tests/__main__.py + PSUTIL_TESTING=1 python -Wa -m coverage run psutil/tests/__main__.py else - PSUTIL_TESTING=1 python -Wa tests/__main__.py + PSUTIL_TESTING=1 python -Wa psutil/tests/__main__.py fi if [ "$PYVER" == "2.7" ] || [ "$PYVER" == "3.6" ]; then # run mem leaks test - PSUTIL_TESTING=1 python -Wa tests/test_memory_leaks.py + PSUTIL_TESTING=1 python -Wa psutil/tests/test_memory_leaks.py # run linter (on Linux only) if [[ "$(uname -s)" != 'Darwin' ]]; then python -m flake8 diff --git a/.coveragerc b/.coveragerc index 28316cfb8..7d3f185f5 100644 --- a/.coveragerc +++ b/.coveragerc @@ -4,7 +4,7 @@ include = *psutil* omit = psutil/_compat.py - tests/* + psutil/tests/* setup.py exclude_lines = enum.IntEnum diff --git a/DEVGUIDE.rst b/DEVGUIDE.rst index df4c9cbbf..2d48ced27 100644 --- a/DEVGUIDE.rst +++ b/DEVGUIDE.rst @@ -99,8 +99,8 @@ Usually the files involved when adding a new functionality are: psutil/_ps{platform}.py # python platform wrapper psutil/_psutil_{platform}.c # C platform extension psutil/_psutil_{platform}.h # C header file - tests/test_process|system.py # main test suite - tests/test_{platform}.py # platform specific test suite + psutil/tests/test_process|system.py # main test suite + psutil/tests/test_{platform}.py # platform specific test suite Typical process occurring when adding a new functionality (API): @@ -109,11 +109,12 @@ Typical process occurring when adding a new functionality (API): (e.g. ``psutil/_pslinux.py``). - if the change requires C, write the C implementation in ``psutil/_psutil_{platform}.c`` (e.g. ``psutil/_psutil_linux.c``). -- write a generic test in ``tests/test_system.py`` or - ``tests/test_process.py``. -- if possible, write a platform specific test in ``tests/test_{platform}.py`` - (e.g. ``test_linux.py``). This usually means testing the return value of the - new feature against a system CLI tool. +- write a generic test in ``psutil/tests/test_system.py`` or + ``psutil/tests/test_process.py``. +- if possible, write a platform specific test in + ``psutil/tests/test_{platform}.py`` (e.g. ``test_linux.py``). + This usually means testing the return value of the new feature against + a system CLI tool. - update doc in ``doc/index.py``. - update ``HISTORY.rst``. - update ``README.rst`` (if necessary). diff --git a/MANIFEST.in b/MANIFEST.in index c422be868..7a92a4e5a 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -79,6 +79,23 @@ include psutil/arch/windows/security.c include psutil/arch/windows/security.h include psutil/arch/windows/services.c include psutil/arch/windows/services.h +include psutil/tests/README.rst +include psutil/tests/__init__.py +include psutil/tests/__main__.py +include psutil/tests/test_aix.py +include psutil/tests/test_bsd.py +include psutil/tests/test_connections.py +include psutil/tests/test_contracts.py +include psutil/tests/test_linux.py +include psutil/tests/test_memory_leaks.py +include psutil/tests/test_misc.py +include psutil/tests/test_osx.py +include psutil/tests/test_posix.py +include psutil/tests/test_process.py +include psutil/tests/test_sunos.py +include psutil/tests/test_system.py +include psutil/tests/test_unicode.py +include psutil/tests/test_windows.py include scripts/battery.py include scripts/cpu_distribution.py include scripts/disk_usage.py @@ -111,21 +128,4 @@ include scripts/top.py include scripts/who.py include scripts/winservices.py include setup.py -include tests/README.rst -include tests/__init__.py -include tests/__main__.py -include tests/test_aix.py -include tests/test_bsd.py -include tests/test_connections.py -include tests/test_contracts.py -include tests/test_linux.py -include tests/test_memory_leaks.py -include tests/test_misc.py -include tests/test_osx.py -include tests/test_posix.py -include tests/test_process.py -include tests/test_sunos.py -include tests/test_system.py -include tests/test_unicode.py -include tests/test_windows.py include tox.ini diff --git a/Makefile b/Makefile index 99355233e..5081a4edd 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ # You can set the variables below from the command line. PYTHON = python -TSCRIPT = tests/__main__.py +TSCRIPT = psutil/tests/__main__.py ARGS = # List of nice-to-have dev libs. DEPS = \ @@ -113,41 +113,41 @@ test: ## Run all tests. test-process: ## Run process-related API tests. ${MAKE} install - $(TEST_PREFIX) $(PYTHON) -m unittest -v tests.test_process + $(TEST_PREFIX) $(PYTHON) -m unittest -v psutil.tests.test_process test-system: ## Run system-related API tests. ${MAKE} install - $(TEST_PREFIX) $(PYTHON) -m unittest -v tests.test_system + $(TEST_PREFIX) $(PYTHON) -m unittest -v psutil.tests.test_system test-misc: ## Run miscellaneous tests. ${MAKE} install - $(TEST_PREFIX) $(PYTHON) tests/test_misc.py + $(TEST_PREFIX) $(PYTHON) psutil/tests/test_misc.py test-unicode: ## Test APIs dealing with strings. ${MAKE} install - $(TEST_PREFIX) $(PYTHON) tests/test_unicode.py + $(TEST_PREFIX) $(PYTHON) psutil/tests/test_unicode.py test-contracts: ## APIs sanity tests. ${MAKE} install - $(TEST_PREFIX) $(PYTHON) tests/test_contracts.py + $(TEST_PREFIX) $(PYTHON) psutil/tests/test_contracts.py test-connections: ## Test net_connections() and Process.connections(). ${MAKE} install - $(TEST_PREFIX) $(PYTHON) tests/test_connections.py + $(TEST_PREFIX) $(PYTHON) psutil/tests/test_connections.py test-posix: ## POSIX specific tests. ${MAKE} install - $(TEST_PREFIX) $(PYTHON) tests/test_posix.py + $(TEST_PREFIX) $(PYTHON) psutil/tests/test_posix.py test-platform: ## Run specific platform tests only. ${MAKE} install - $(TEST_PREFIX) $(PYTHON) tests/test_`$(PYTHON) -c 'import psutil; print([x.lower() for x in ("LINUX", "BSD", "OSX", "SUNOS", "WINDOWS", "AIX") if getattr(psutil, x)][0])'`.py + $(TEST_PREFIX) $(PYTHON) psutil/tests/test_`$(PYTHON) -c 'import psutil; print([x.lower() for x in ("LINUX", "BSD", "OSX", "SUNOS", "WINDOWS", "AIX") if getattr(psutil, x)][0])'`.py test-memleaks: ## Memory leak tests. ${MAKE} install - $(TEST_PREFIX) $(PYTHON) tests/test_memory_leaks.py + $(TEST_PREFIX) $(PYTHON) psutil/tests/test_memory_leaks.py -test-by-name: ## e.g. make test-by-name ARGS=tests.test_system.TestSystemAPIs +test-by-name: ## e.g. make test-by-name ARGS=psutil.tests.test_system.TestSystemAPIs ${MAKE} install @$(TEST_PREFIX) $(PYTHON) -m unittest -v $(ARGS) diff --git a/appveyor.yml b/appveyor.yml index 2ceb63f79..28f5f7f6f 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -83,7 +83,7 @@ build: off test_script: - "%WITH_COMPILER% %PYTHON%/python -V" - - "set PYTHONWARNINGS=all && set PSUTIL_TESTING=1 && set PSUTIL_DEBUG=1 && %WITH_COMPILER% %PYTHON%/python tests/__main__.py" + - "set PYTHONWARNINGS=all && set PSUTIL_TESTING=1 && set PSUTIL_DEBUG=1 && %WITH_COMPILER% %PYTHON%/python psutil/tests/__main__.py" after_test: - "%WITH_COMPILER% %PYTHON%/python setup.py bdist_wheel" @@ -112,12 +112,12 @@ only_commits: psutil/_psutil_windows.* psutil/_pswindows.py psutil/arch/windows/* - tests/__init__.py - tests/__main__.py - tests/test_memory_leaks.py - tests/test_misc.py - tests/test_process.py - tests/test_system.py - tests/test_windows.py + psutil/tests/__init__.py + psutil/tests/__main__.py + psutil/tests/test_memory_leaks.py + psutil/tests/test_misc.py + psutil/tests/test_process.py + psutil/tests/test_system.py + psutil/tests/test_windows.py scripts/* setup.py diff --git a/docs/index.rst b/docs/index.rst index 5ea03ab82..4f77b2e49 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -2605,7 +2605,12 @@ FAQs Running tests ============= -First, obtain a copy of the source code. Then run tests using the command:: +There are two ways of running tests. If psutil is already installed use:: + + $ python -m psutil.tests + +You can use this method as a quick way to make sure psutil fully works on your +platform. If you have a copy of the source code you can also use:: $ make test diff --git a/tests/README.rst b/psutil/tests/README.rst similarity index 62% rename from tests/README.rst rename to psutil/tests/README.rst index a02b88134..515abf772 100644 --- a/tests/README.rst +++ b/psutil/tests/README.rst @@ -1,7 +1,14 @@ Instructions for running tests ============================== -* To run tests:: +* There are two ways of running tests. As a "user", if psutil is already + installed and you just want to test it works:: + + python -m psutil.tests --install-deps # install test deps + python -m psutil.tests + + As a "developer", if you have a copy of the source code and you whish to hack + on psutil:: make setup-dev-env # install test deps (+ other things) make test diff --git a/tests/__init__.py b/psutil/tests/__init__.py similarity index 99% rename from tests/__init__.py rename to psutil/tests/__init__.py index f59cd846e..499240fb2 100644 --- a/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -142,7 +142,7 @@ # --- paths -ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) +ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..')) SCRIPTS_DIR = os.path.join(ROOT_DIR, 'scripts') HERE = os.path.abspath(os.path.dirname(__file__)) @@ -788,7 +788,7 @@ def get_suite(): suite = unittest.TestSuite() for tm in testmods: # ...so that the full test paths are printed on screen - tm = "tests.%s" % tm + tm = "psutil.tests.%s" % tm suite.addTest(unittest.defaultTestLoader.loadTestsFromName(tm)) return suite diff --git a/tests/__main__.py b/psutil/tests/__main__.py similarity index 90% rename from tests/__main__.py rename to psutil/tests/__main__.py index 97690afeb..62fe07428 100755 --- a/tests/__main__.py +++ b/psutil/tests/__main__.py @@ -5,7 +5,9 @@ # found in the LICENSE file. """ -Run unit tests. +Run unit tests. This is invoked by: + +$ python -m psutil.tests """ import contextlib @@ -19,8 +21,8 @@ except ImportError: from urllib2 import urlopen -from tests import PYTHON_EXE -from tests import run_suite +from psutil.tests import PYTHON_EXE +from psutil.tests import run_suite HERE = os.path.abspath(os.path.dirname(__file__)) @@ -69,7 +71,7 @@ def install_test_deps(deps=None): def main(): - usage = "%s -m tests [opts]" % PYTHON_EXE + usage = "%s -m psutil.tests [opts]" % PYTHON_EXE parser = optparse.OptionParser(usage=usage, description="run unit tests") parser.add_option("-i", "--install-deps", action="store_true", default=False, @@ -84,7 +86,7 @@ def main(): try: __import__(dep.split("==")[0]) except ImportError: - sys.exit("%r lib is not installed; run %s -m tests " + sys.exit("%r lib is not installed; run %s -m psutil.tests " "--install-deps" % (dep, PYTHON_EXE)) run_suite() diff --git a/tests/test_aix.py b/psutil/tests/test_aix.py similarity index 93% rename from tests/test_aix.py rename to psutil/tests/test_aix.py index 8370c3830..7a8a4c334 100755 --- a/tests/test_aix.py +++ b/psutil/tests/test_aix.py @@ -11,9 +11,9 @@ import re from psutil import AIX -from tests import run_test_module_by_name -from tests import sh -from tests import unittest +from psutil.tests import run_test_module_by_name +from psutil.tests import sh +from psutil.tests import unittest import psutil @@ -38,9 +38,9 @@ def test_virtual_memory(self): psutil_result = psutil.virtual_memory() - # MEMORY_TOLERANCE from tests is not enough. For some reason we're - # seeing differences of ~1.2 MB. 2 MB is still a good tolerance when - # compared to GBs. + # MEMORY_TOLERANCE from psutil.tests is not enough. For some reason + # we're seeing differences of ~1.2 MB. 2 MB is still a good tolerance + # when compared to GBs. MEMORY_TOLERANCE = 2 * KB * KB # 2 MB self.assertEqual(psutil_result.total, total) self.assertAlmostEqual( diff --git a/tests/test_bsd.py b/psutil/tests/test_bsd.py similarity index 98% rename from tests/test_bsd.py rename to psutil/tests/test_bsd.py index cc9c058d3..d3868ada1 100755 --- a/tests/test_bsd.py +++ b/psutil/tests/test_bsd.py @@ -20,15 +20,15 @@ from psutil import FREEBSD from psutil import NETBSD from psutil import OPENBSD -from tests import get_test_subprocess -from tests import HAS_BATTERY -from tests import MEMORY_TOLERANCE -from tests import reap_children -from tests import retry_before_failing -from tests import run_test_module_by_name -from tests import sh -from tests import unittest -from tests import which +from psutil.tests import get_test_subprocess +from psutil.tests import HAS_BATTERY +from psutil.tests import MEMORY_TOLERANCE +from psutil.tests import reap_children +from psutil.tests import retry_before_failing +from psutil.tests import run_test_module_by_name +from psutil.tests import sh +from psutil.tests import unittest +from psutil.tests import which if BSD: diff --git a/tests/test_connections.py b/psutil/tests/test_connections.py similarity index 96% rename from tests/test_connections.py rename to psutil/tests/test_connections.py index c61e0c2c7..176e26648 100755 --- a/tests/test_connections.py +++ b/psutil/tests/test_connections.py @@ -26,25 +26,25 @@ from psutil import WINDOWS from psutil._common import supports_ipv6 from psutil._compat import PY3 -from tests import AF_UNIX -from tests import bind_socket -from tests import bind_unix_socket -from tests import check_connection_ntuple -from tests import create_sockets -from tests import get_free_port -from tests import HAS_CONNECTIONS_UNIX -from tests import pyrun -from tests import reap_children -from tests import run_test_module_by_name -from tests import safe_rmpath -from tests import skip_on_access_denied -from tests import tcp_socketpair -from tests import TESTFN -from tests import TRAVIS -from tests import unittest -from tests import unix_socket_path -from tests import unix_socketpair -from tests import wait_for_file +from psutil.tests import AF_UNIX +from psutil.tests import bind_socket +from psutil.tests import bind_unix_socket +from psutil.tests import check_connection_ntuple +from psutil.tests import create_sockets +from psutil.tests import get_free_port +from psutil.tests import HAS_CONNECTIONS_UNIX +from psutil.tests import pyrun +from psutil.tests import reap_children +from psutil.tests import run_test_module_by_name +from psutil.tests import safe_rmpath +from psutil.tests import skip_on_access_denied +from psutil.tests import tcp_socketpair +from psutil.tests import TESTFN +from psutil.tests import TRAVIS +from psutil.tests import unittest +from psutil.tests import unix_socket_path +from psutil.tests import unix_socketpair +from psutil.tests import wait_for_file thisproc = psutil.Process() @@ -471,7 +471,7 @@ def test_multi_sockets_procs(self): fname = os.path.realpath(TESTFN) + str(i) src = textwrap.dedent("""\ import time, os - from tests import create_sockets + from psutil.tests import create_sockets with create_sockets(): with open('%s', 'w') as f: f.write(str(os.getpid())) diff --git a/tests/test_contracts.py b/psutil/tests/test_contracts.py similarity index 97% rename from tests/test_contracts.py rename to psutil/tests/test_contracts.py index f07c2624e..0dfb3e2a8 100755 --- a/tests/test_contracts.py +++ b/psutil/tests/test_contracts.py @@ -28,22 +28,22 @@ from psutil import SUNOS from psutil import WINDOWS from psutil._compat import long -from tests import bind_unix_socket -from tests import check_connection_ntuple -from tests import get_kernel_version -from tests import HAS_CONNECTIONS_UNIX -from tests import HAS_RLIMIT -from tests import HAS_SENSORS_FANS -from tests import HAS_SENSORS_TEMPERATURES -from tests import is_namedtuple -from tests import run_test_module_by_name -from tests import safe_rmpath -from tests import skip_on_access_denied -from tests import TESTFN -from tests import unittest -from tests import unix_socket_path -from tests import VALID_PROC_STATUSES -from tests import warn +from psutil.tests import bind_unix_socket +from psutil.tests import check_connection_ntuple +from psutil.tests import get_kernel_version +from psutil.tests import HAS_CONNECTIONS_UNIX +from psutil.tests import HAS_RLIMIT +from psutil.tests import HAS_SENSORS_FANS +from psutil.tests import HAS_SENSORS_TEMPERATURES +from psutil.tests import is_namedtuple +from psutil.tests import run_test_module_by_name +from psutil.tests import safe_rmpath +from psutil.tests import skip_on_access_denied +from psutil.tests import TESTFN +from psutil.tests import unittest +from psutil.tests import unix_socket_path +from psutil.tests import VALID_PROC_STATUSES +from psutil.tests import warn import psutil diff --git a/tests/test_linux.py b/psutil/tests/test_linux.py similarity index 99% rename from tests/test_linux.py rename to psutil/tests/test_linux.py index 7cc5a01ac..6ba17b254 100755 --- a/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -27,26 +27,26 @@ from psutil import LINUX from psutil._compat import PY3 from psutil._compat import u -from tests import call_until -from tests import HAS_BATTERY -from tests import HAS_CPU_FREQ -from tests import HAS_RLIMIT -from tests import MEMORY_TOLERANCE -from tests import mock -from tests import PYPY -from tests import pyrun -from tests import reap_children -from tests import reload_module -from tests import retry_before_failing -from tests import run_test_module_by_name -from tests import safe_rmpath -from tests import sh -from tests import skip_on_not_implemented -from tests import TESTFN -from tests import ThreadTask -from tests import TRAVIS -from tests import unittest -from tests import which +from psutil.tests import call_until +from psutil.tests import HAS_BATTERY +from psutil.tests import HAS_CPU_FREQ +from psutil.tests import HAS_RLIMIT +from psutil.tests import MEMORY_TOLERANCE +from psutil.tests import mock +from psutil.tests import PYPY +from psutil.tests import pyrun +from psutil.tests import reap_children +from psutil.tests import reload_module +from psutil.tests import retry_before_failing +from psutil.tests import run_test_module_by_name +from psutil.tests import safe_rmpath +from psutil.tests import sh +from psutil.tests import skip_on_not_implemented +from psutil.tests import TESTFN +from psutil.tests import ThreadTask +from psutil.tests import TRAVIS +from psutil.tests import unittest +from psutil.tests import which HERE = os.path.abspath(os.path.dirname(__file__)) diff --git a/tests/test_memory_leaks.py b/psutil/tests/test_memory_leaks.py similarity index 95% rename from tests/test_memory_leaks.py rename to psutil/tests/test_memory_leaks.py index 53723c7bd..680fe7803 100755 --- a/tests/test_memory_leaks.py +++ b/psutil/tests/test_memory_leaks.py @@ -31,26 +31,26 @@ from psutil import SUNOS from psutil import WINDOWS from psutil._compat import xrange -from tests import create_sockets -from tests import get_test_subprocess -from tests import HAS_CPU_AFFINITY -from tests import HAS_CPU_FREQ -from tests import HAS_ENVIRON -from tests import HAS_IONICE -from tests import HAS_MEMORY_MAPS -from tests import HAS_PROC_CPU_NUM -from tests import HAS_PROC_IO_COUNTERS -from tests import HAS_RLIMIT -from tests import HAS_SENSORS_BATTERY -from tests import HAS_SENSORS_FANS -from tests import HAS_SENSORS_TEMPERATURES -from tests import reap_children -from tests import run_test_module_by_name -from tests import safe_rmpath -from tests import skip_on_access_denied -from tests import TESTFN -from tests import TRAVIS -from tests import unittest +from psutil.tests import create_sockets +from psutil.tests import get_test_subprocess +from psutil.tests import HAS_CPU_AFFINITY +from psutil.tests import HAS_CPU_FREQ +from psutil.tests import HAS_ENVIRON +from psutil.tests import HAS_IONICE +from psutil.tests import HAS_MEMORY_MAPS +from psutil.tests import HAS_PROC_CPU_NUM +from psutil.tests import HAS_PROC_IO_COUNTERS +from psutil.tests import HAS_RLIMIT +from psutil.tests import HAS_SENSORS_BATTERY +from psutil.tests import HAS_SENSORS_FANS +from psutil.tests import HAS_SENSORS_TEMPERATURES +from psutil.tests import reap_children +from psutil.tests import run_test_module_by_name +from psutil.tests import safe_rmpath +from psutil.tests import skip_on_access_denied +from psutil.tests import TESTFN +from psutil.tests import TRAVIS +from psutil.tests import unittest LOOPS = 1000 diff --git a/tests/test_misc.py b/psutil/tests/test_misc.py similarity index 94% rename from tests/test_misc.py rename to psutil/tests/test_misc.py index 3dbd49d17..1d9067e77 100755 --- a/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -27,48 +27,48 @@ from psutil._common import supports_ipv6 from psutil._common import wrap_numbers from psutil._compat import PY3 -from tests import APPVEYOR -from tests import bind_socket -from tests import bind_unix_socket -from tests import call_until -from tests import chdir -from tests import create_proc_children_pair -from tests import create_sockets -from tests import create_zombie_proc -from tests import DEVNULL -from tests import get_free_port -from tests import get_test_subprocess -from tests import HAS_BATTERY -from tests import HAS_CONNECTIONS_UNIX -from tests import HAS_MEMORY_FULL_INFO -from tests import HAS_MEMORY_MAPS -from tests import HAS_SENSORS_BATTERY -from tests import HAS_SENSORS_FANS -from tests import HAS_SENSORS_TEMPERATURES -from tests import import_module_by_path -from tests import is_namedtuple -from tests import mock -from tests import PYTHON_EXE -from tests import reap_children -from tests import reload_module -from tests import retry -from tests import ROOT_DIR -from tests import run_test_module_by_name -from tests import safe_mkdir -from tests import safe_rmpath -from tests import SCRIPTS_DIR -from tests import sh -from tests import tcp_socketpair -from tests import TESTFN -from tests import TOX -from tests import TRAVIS -from tests import unittest -from tests import unix_socket_path -from tests import unix_socketpair -from tests import wait_for_file -from tests import wait_for_pid +from psutil.tests import APPVEYOR +from psutil.tests import bind_socket +from psutil.tests import bind_unix_socket +from psutil.tests import call_until +from psutil.tests import chdir +from psutil.tests import create_proc_children_pair +from psutil.tests import create_sockets +from psutil.tests import create_zombie_proc +from psutil.tests import DEVNULL +from psutil.tests import get_free_port +from psutil.tests import get_test_subprocess +from psutil.tests import HAS_BATTERY +from psutil.tests import HAS_CONNECTIONS_UNIX +from psutil.tests import HAS_MEMORY_FULL_INFO +from psutil.tests import HAS_MEMORY_MAPS +from psutil.tests import HAS_SENSORS_BATTERY +from psutil.tests import HAS_SENSORS_FANS +from psutil.tests import HAS_SENSORS_TEMPERATURES +from psutil.tests import import_module_by_path +from psutil.tests import is_namedtuple +from psutil.tests import mock +from psutil.tests import PYTHON_EXE +from psutil.tests import reap_children +from psutil.tests import reload_module +from psutil.tests import retry +from psutil.tests import ROOT_DIR +from psutil.tests import run_test_module_by_name +from psutil.tests import safe_mkdir +from psutil.tests import safe_rmpath +from psutil.tests import SCRIPTS_DIR +from psutil.tests import sh +from psutil.tests import tcp_socketpair +from psutil.tests import TESTFN +from psutil.tests import TOX +from psutil.tests import TRAVIS +from psutil.tests import unittest +from psutil.tests import unix_socket_path +from psutil.tests import unix_socketpair +from psutil.tests import wait_for_file +from psutil.tests import wait_for_pid import psutil -import tests +import psutil.tests # =================================================================== @@ -859,7 +859,7 @@ def tearDown(self): def test_wait_for_pid(self): wait_for_pid(os.getpid()) nopid = max(psutil.pids()) + 99999 - with mock.patch('tests.retry.__iter__', return_value=iter([0])): + with mock.patch('psutil.tests.retry.__iter__', return_value=iter([0])): self.assertRaises(psutil.NoSuchProcess, wait_for_pid, nopid) def test_wait_for_file(self): @@ -875,7 +875,7 @@ def test_wait_for_file_empty(self): assert not os.path.exists(TESTFN) def test_wait_for_file_no_file(self): - with mock.patch('tests.retry.__iter__', return_value=iter([0])): + with mock.patch('psutil.tests.retry.__iter__', return_value=iter([0])): self.assertRaises(IOError, wait_for_file, TESTFN) def test_wait_for_file_no_delete(self): @@ -914,7 +914,7 @@ def test_safe_rmpath(self): safe_rmpath(TESTFN) assert not os.path.exists(TESTFN) # test other exceptions are raised - with mock.patch('tests.os.stat', + with mock.patch('psutil.tests.os.stat', side_effect=OSError(errno.EINVAL, "")) as m: with self.assertRaises(OSError): safe_rmpath(TESTFN) @@ -936,8 +936,8 @@ def test_reap_children(self): assert p.is_running() reap_children() assert not p.is_running() - assert not tests._pids_started - assert not tests._subprocesses_started + assert not psutil.tests._pids_started + assert not psutil.tests._subprocesses_started def test_create_proc_children_pair(self): p1, p2 = create_proc_children_pair() @@ -955,8 +955,8 @@ def test_create_proc_children_pair(self): reap_children() assert not p1.is_running() assert not p2.is_running() - assert not tests._pids_started - assert not tests._subprocesses_started + assert not psutil.tests._pids_started + assert not psutil.tests._subprocesses_started @unittest.skipIf(not POSIX, "POSIX only") def test_create_zombie_proc(self): diff --git a/tests/test_osx.py b/psutil/tests/test_osx.py similarity index 96% rename from tests/test_osx.py rename to psutil/tests/test_osx.py index 4e8b2e470..bcb2ba4e1 100755 --- a/tests/test_osx.py +++ b/psutil/tests/test_osx.py @@ -12,15 +12,15 @@ import psutil from psutil import OSX -from tests import create_zombie_proc -from tests import get_test_subprocess -from tests import HAS_BATTERY -from tests import MEMORY_TOLERANCE -from tests import reap_children -from tests import retry_before_failing -from tests import run_test_module_by_name -from tests import sh -from tests import unittest +from psutil.tests import create_zombie_proc +from psutil.tests import get_test_subprocess +from psutil.tests import HAS_BATTERY +from psutil.tests import MEMORY_TOLERANCE +from psutil.tests import reap_children +from psutil.tests import retry_before_failing +from psutil.tests import run_test_module_by_name +from psutil.tests import sh +from psutil.tests import unittest PAGESIZE = os.sysconf("SC_PAGE_SIZE") if OSX else None diff --git a/tests/test_posix.py b/psutil/tests/test_posix.py similarity index 96% rename from tests/test_posix.py rename to psutil/tests/test_posix.py index 382ebade6..e9a6f5f63 100755 --- a/tests/test_posix.py +++ b/psutil/tests/test_posix.py @@ -24,20 +24,20 @@ from psutil import POSIX from psutil import SUNOS from psutil._compat import PY3 -from tests import APPVEYOR -from tests import get_kernel_version -from tests import get_test_subprocess -from tests import mock -from tests import PYTHON_EXE -from tests import reap_children -from tests import retry_before_failing -from tests import run_test_module_by_name -from tests import sh -from tests import skip_on_access_denied -from tests import TRAVIS -from tests import unittest -from tests import wait_for_pid -from tests import which +from psutil.tests import APPVEYOR +from psutil.tests import get_kernel_version +from psutil.tests import get_test_subprocess +from psutil.tests import mock +from psutil.tests import PYTHON_EXE +from psutil.tests import reap_children +from psutil.tests import retry_before_failing +from psutil.tests import run_test_module_by_name +from psutil.tests import sh +from psutil.tests import skip_on_access_denied +from psutil.tests import TRAVIS +from psutil.tests import unittest +from psutil.tests import wait_for_pid +from psutil.tests import which def ps(cmd): diff --git a/tests/test_process.py b/psutil/tests/test_process.py similarity index 97% rename from tests/test_process.py rename to psutil/tests/test_process.py index 1c0d6a8cd..33557b051 100755 --- a/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -33,40 +33,40 @@ from psutil import WINDOWS from psutil._compat import long from psutil._compat import PY3 -from tests import APPVEYOR -from tests import call_until -from tests import copyload_shared_lib -from tests import create_exe -from tests import create_proc_children_pair -from tests import create_zombie_proc -from tests import enum -from tests import get_test_subprocess -from tests import get_winver -from tests import HAS_CPU_AFFINITY -from tests import HAS_ENVIRON -from tests import HAS_IONICE -from tests import HAS_MEMORY_MAPS -from tests import HAS_PROC_CPU_NUM -from tests import HAS_PROC_IO_COUNTERS -from tests import HAS_RLIMIT -from tests import HAS_THREADS -from tests import mock -from tests import PYPY -from tests import PYTHON_EXE -from tests import reap_children -from tests import retry_before_failing -from tests import run_test_module_by_name -from tests import safe_rmpath -from tests import sh -from tests import skip_on_access_denied -from tests import skip_on_not_implemented -from tests import TESTFILE_PREFIX -from tests import TESTFN -from tests import ThreadTask -from tests import TRAVIS -from tests import unittest -from tests import wait_for_pid -from tests import WIN_VISTA +from psutil.tests import APPVEYOR +from psutil.tests import call_until +from psutil.tests import copyload_shared_lib +from psutil.tests import create_exe +from psutil.tests import create_proc_children_pair +from psutil.tests import create_zombie_proc +from psutil.tests import enum +from psutil.tests import get_test_subprocess +from psutil.tests import get_winver +from psutil.tests import HAS_CPU_AFFINITY +from psutil.tests import HAS_ENVIRON +from psutil.tests import HAS_IONICE +from psutil.tests import HAS_MEMORY_MAPS +from psutil.tests import HAS_PROC_CPU_NUM +from psutil.tests import HAS_PROC_IO_COUNTERS +from psutil.tests import HAS_RLIMIT +from psutil.tests import HAS_THREADS +from psutil.tests import mock +from psutil.tests import PYPY +from psutil.tests import PYTHON_EXE +from psutil.tests import reap_children +from psutil.tests import retry_before_failing +from psutil.tests import run_test_module_by_name +from psutil.tests import safe_rmpath +from psutil.tests import sh +from psutil.tests import skip_on_access_denied +from psutil.tests import skip_on_not_implemented +from psutil.tests import TESTFILE_PREFIX +from psutil.tests import TESTFN +from psutil.tests import ThreadTask +from psutil.tests import TRAVIS +from psutil.tests import unittest +from psutil.tests import wait_for_pid +from psutil.tests import WIN_VISTA # =================================================================== diff --git a/tests/test_sunos.py b/psutil/tests/test_sunos.py similarity index 91% rename from tests/test_sunos.py rename to psutil/tests/test_sunos.py index e266fb64e..ea9afcde0 100755 --- a/tests/test_sunos.py +++ b/psutil/tests/test_sunos.py @@ -10,9 +10,9 @@ import psutil from psutil import SUNOS -from tests import run_test_module_by_name -from tests import sh -from tests import unittest +from psutil.tests import run_test_module_by_name +from psutil.tests import sh +from psutil.tests import unittest @unittest.skipIf(not SUNOS, "SUNOS only") diff --git a/tests/test_system.py b/psutil/tests/test_system.py similarity index 97% rename from tests/test_system.py rename to psutil/tests/test_system.py index 6eb891d4f..20b132a91 100755 --- a/tests/test_system.py +++ b/psutil/tests/test_system.py @@ -30,26 +30,26 @@ from psutil import SUNOS from psutil import WINDOWS from psutil._compat import long -from tests import APPVEYOR -from tests import ASCII_FS -from tests import check_net_address -from tests import DEVNULL -from tests import enum -from tests import get_test_subprocess -from tests import HAS_BATTERY -from tests import HAS_CPU_FREQ -from tests import HAS_SENSORS_BATTERY -from tests import HAS_SENSORS_FANS -from tests import HAS_SENSORS_TEMPERATURES -from tests import mock -from tests import reap_children -from tests import retry_before_failing -from tests import run_test_module_by_name -from tests import safe_rmpath -from tests import TESTFN -from tests import TESTFN_UNICODE -from tests import TRAVIS -from tests import unittest +from psutil.tests import APPVEYOR +from psutil.tests import ASCII_FS +from psutil.tests import check_net_address +from psutil.tests import DEVNULL +from psutil.tests import enum +from psutil.tests import get_test_subprocess +from psutil.tests import HAS_BATTERY +from psutil.tests import HAS_CPU_FREQ +from psutil.tests import HAS_SENSORS_BATTERY +from psutil.tests import HAS_SENSORS_FANS +from psutil.tests import HAS_SENSORS_TEMPERATURES +from psutil.tests import mock +from psutil.tests import reap_children +from psutil.tests import retry_before_failing +from psutil.tests import run_test_module_by_name +from psutil.tests import safe_rmpath +from psutil.tests import TESTFN +from psutil.tests import TESTFN_UNICODE +from psutil.tests import TRAVIS +from psutil.tests import unittest # =================================================================== diff --git a/tests/test_unicode.py b/psutil/tests/test_unicode.py similarity index 93% rename from tests/test_unicode.py rename to psutil/tests/test_unicode.py index ea1a6900a..6383c9bec 100755 --- a/tests/test_unicode.py +++ b/psutil/tests/test_unicode.py @@ -64,28 +64,28 @@ from psutil import WINDOWS from psutil._compat import PY3 from psutil._compat import u -from tests import APPVEYOR -from tests import ASCII_FS -from tests import bind_unix_socket -from tests import chdir -from tests import copyload_shared_lib -from tests import create_exe -from tests import get_test_subprocess -from tests import HAS_CONNECTIONS_UNIX -from tests import HAS_ENVIRON -from tests import HAS_MEMORY_MAPS -from tests import mock -from tests import reap_children -from tests import run_test_module_by_name -from tests import safe_mkdir -from tests import safe_rmpath as _safe_rmpath -from tests import skip_on_access_denied -from tests import TESTFILE_PREFIX -from tests import TESTFN -from tests import TESTFN_UNICODE -from tests import TRAVIS -from tests import unittest -from tests import unix_socket_path +from psutil.tests import APPVEYOR +from psutil.tests import ASCII_FS +from psutil.tests import bind_unix_socket +from psutil.tests import chdir +from psutil.tests import copyload_shared_lib +from psutil.tests import create_exe +from psutil.tests import get_test_subprocess +from psutil.tests import HAS_CONNECTIONS_UNIX +from psutil.tests import HAS_ENVIRON +from psutil.tests import HAS_MEMORY_MAPS +from psutil.tests import mock +from psutil.tests import reap_children +from psutil.tests import run_test_module_by_name +from psutil.tests import safe_mkdir +from psutil.tests import safe_rmpath as _safe_rmpath +from psutil.tests import skip_on_access_denied +from psutil.tests import TESTFILE_PREFIX +from psutil.tests import TESTFN +from psutil.tests import TESTFN_UNICODE +from psutil.tests import TRAVIS +from psutil.tests import unittest +from psutil.tests import unix_socket_path import psutil diff --git a/tests/test_windows.py b/psutil/tests/test_windows.py similarity index 98% rename from tests/test_windows.py rename to psutil/tests/test_windows.py index 236f2aefc..32c46f67a 100755 --- a/tests/test_windows.py +++ b/psutil/tests/test_windows.py @@ -21,15 +21,15 @@ import psutil from psutil import WINDOWS -from tests import APPVEYOR -from tests import get_test_subprocess -from tests import HAS_BATTERY -from tests import mock -from tests import reap_children -from tests import retry_before_failing -from tests import run_test_module_by_name -from tests import sh -from tests import unittest +from psutil.tests import APPVEYOR +from psutil.tests import get_test_subprocess +from psutil.tests import HAS_BATTERY +from psutil.tests import mock +from psutil.tests import reap_children +from psutil.tests import retry_before_failing +from psutil.tests import run_test_module_by_name +from psutil.tests import sh +from psutil.tests import unittest with warnings.catch_warnings(): warnings.simplefilter("ignore") diff --git a/scripts/internal/winmake.py b/scripts/internal/winmake.py index de58fbfca..548f7a8ed 100755 --- a/scripts/internal/winmake.py +++ b/scripts/internal/winmake.py @@ -355,7 +355,7 @@ def test_process(): """Run process tests""" install() test_setup() - sh("%s -m unittest -v tests.test_process" % PYTHON) + sh("%s -m unittest -v psutil.tests.test_process" % PYTHON) @cmd @@ -363,7 +363,7 @@ def test_system(): """Run system tests""" install() test_setup() - sh("%s -m unittest -v tests.test_system" % PYTHON) + sh("%s -m unittest -v psutil.tests.test_system" % PYTHON) @cmd @@ -371,7 +371,7 @@ def test_platform(): """Run windows only tests""" install() test_setup() - sh("%s -m unittest -v tests.test_windows" % PYTHON) + sh("%s -m unittest -v psutil.tests.test_windows" % PYTHON) @cmd @@ -379,7 +379,7 @@ def test_misc(): """Run misc tests""" install() test_setup() - sh("%s -m unittest -v tests.test_misc" % PYTHON) + sh("%s -m unittest -v psutil.tests.test_misc" % PYTHON) @cmd @@ -387,7 +387,7 @@ def test_unicode(): """Run unicode tests""" install() test_setup() - sh("%s -m unittest -v tests.test_unicode" % PYTHON) + sh("%s -m unittest -v psutil.tests.test_unicode" % PYTHON) @cmd @@ -395,7 +395,7 @@ def test_connections(): """Run connections tests""" install() test_setup() - sh("%s -m unittest -v tests.test_connections" % PYTHON) + sh("%s -m unittest -v psutil.tests.test_connections" % PYTHON) @cmd @@ -403,7 +403,7 @@ def test_contracts(): """Run contracts tests""" install() test_setup() - sh("%s -m unittest -v tests.test_contracts" % PYTHON) + sh("%s -m unittest -v psutil.tests.test_contracts" % PYTHON) @cmd diff --git a/setup.py b/setup.py index f03ab4570..61056f5f1 100755 --- a/setup.py +++ b/setup.py @@ -279,7 +279,7 @@ def main(): url='https://github.com/giampaolo/psutil', platforms='Platform Independent', license='BSD', - packages=['psutil'], + packages=['psutil', 'psutil.tests'], ext_modules=extensions, # see: python setup.py register --list-classifiers classifiers=[ @@ -327,7 +327,7 @@ def main(): if setuptools is not None: kwargs.update( python_requires=">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*", - test_suite="tests.get_suite", + test_suite="psutil.tests.get_suite", tests_require=[ 'ipaddress; python_version < "3.3"', 'mock; python_version < "3.3"', diff --git a/tox.ini b/tox.ini index 7ec895af0..61d63eee7 100644 --- a/tox.ini +++ b/tox.ini @@ -18,7 +18,7 @@ deps = setenv = TOX = 1 -commands = python tests/__main__.py +commands = python psutil/tests/__main__.py usedevelop = True From 66edd7ade36c871a15f2cb230ae1c054f842555a Mon Sep 17 00:00:00 2001 From: alxchk Date: Sun, 11 Mar 2018 18:48:13 +0200 Subject: [PATCH 019/182] Try to extract argv[] from process address space first (#1220) * Try to extract argv[] from process address space first * Add comments and OOM exception * Handle the case with empty array --- psutil/_psutil_sunos.c | 83 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 82 insertions(+), 1 deletion(-) diff --git a/psutil/_psutil_sunos.c b/psutil/_psutil_sunos.c index c6673642c..4d38a3423 100644 --- a/psutil/_psutil_sunos.c +++ b/psutil/_psutil_sunos.c @@ -116,6 +116,58 @@ psutil_proc_basic_info(PyObject *self, PyObject *args) { ); } +/* + * Join array of C strings to C string with delemiter dm. + * Omit empty records. + */ +static int +cstrings_array_to_string(char **joined, char ** array, size_t count, char dm) { + int i; + size_t total_length = 0; + size_t item_length = 0; + char *result = NULL; + char *last = NULL; + + if (!array || !joined) + return 0; + + for (i=0; i 0) { + py_args = PyUnicode_DecodeFSDefault(argv_plain); + free(argv_plain); + } else if (joined < 0) { + goto error; + } + + psutil_free_cstrings_array(argv, argc); + } + } + + /* If we can't read process memory or can't decode the result + * then return args from /proc. */ + if (!py_args) + py_args = PyUnicode_DecodeFSDefault(info.pr_psargs); + + /* Both methods has been failed. */ if (!py_args) goto error; + py_retlist = Py_BuildValue("OO", py_name, py_args); if (!py_retlist) goto error; + Py_DECREF(py_name); Py_DECREF(py_args); return py_retlist; From 4241713ea670c042e8a871226d3c45959826cd14 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 11 Mar 2018 17:53:15 +0100 Subject: [PATCH 020/182] give credits to @alxchk for #1220; bump up version --- CREDITS | 4 +++- HISTORY.rst | 6 ++++++ psutil/__init__.py | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/CREDITS b/CREDITS index 798b36e39..e0d748dad 100644 --- a/CREDITS +++ b/CREDITS @@ -486,7 +486,7 @@ I: 1042, 1079 N: Oleksii Shevchuk W: https://github.com/alxchk -I: 1077, 1093, 1091 +I: 1077, 1093, 1091, 1220 N: Prodesire W: https://github.com/Prodesire @@ -519,3 +519,5 @@ I: 1169 N: Dan Vinakovsky W: https://github.com/hexaclock I: 1216 + +N diff --git a/HISTORY.rst b/HISTORY.rst index 463860c25..bf378ed80 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -5,6 +5,12 @@ XXXX-XX-XX +**Enhancements** + +- 1120_: [SunOS] cmdline() could be truncated at the 15th character when + reading it from /proc. An extra effort is made by reading it from process + address space first. + **Bug fixes** - 1216_: fix compatibility with python 2.6 on Windows (patch by Dan Vinakovsky) diff --git a/psutil/__init__.py b/psutil/__init__.py index 97767f116..a11988ba2 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -218,7 +218,7 @@ ] __all__.extend(_psplatform.__extra__all__) __author__ = "Giampaolo Rodola'" -__version__ = "5.4.3" +__version__ = "5.4.4" version_info = tuple([int(num) for num in __version__.split('.')]) AF_LINK = _psplatform.AF_LINK POWER_TIME_UNLIMITED = _common.POWER_TIME_UNLIMITED From 41dadf682146fb88249a15d5997d609b4cec3635 Mon Sep 17 00:00:00 2001 From: stswandering <294912115@qq.com> Date: Tue, 13 Mar 2018 16:39:26 +0800 Subject: [PATCH 021/182] =?UTF-8?q?cpu=5Ftimes()=20GetSystemTimes=20functi?= =?UTF-8?q?on=20double=20dword=20convert=20to=20=5F=5Fint64,t=E2=80=A6=20(?= =?UTF-8?q?#1243)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * cpu_times() GetSystemTimes function double dword convert to __int64,then double,instead of float. * use double instead of a new conversion logic. * Py_BuildValue change f to d * Py_BuildValue change f to d --- psutil/_psutil_windows.c | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/psutil/_psutil_windows.c b/psutil/_psutil_windows.c index 81d1b4a06..f164168af 100644 --- a/psutil/_psutil_windows.c +++ b/psutil/_psutil_windows.c @@ -51,8 +51,8 @@ #define MALLOC(x) HeapAlloc(GetProcessHeap(), 0, (x)) #define FREE(x) HeapFree(GetProcessHeap(), 0, (x)) -#define LO_T ((float)1e-7) -#define HI_T (LO_T*4294967296.0) +#define LO_T 1e-7 +#define HI_T 429.4967296 #define BYTESWAP_USHORT(x) ((((USHORT)(x) << 8) | ((USHORT)(x) >> 8)) & 0xffff) #ifndef AF_INET6 #define AF_INET6 23 @@ -899,7 +899,6 @@ psutil_virtual_mem(PyObject *self, PyObject *args) { memInfo.ullAvailVirtual); // avail virtual } - /* * Retrieves system CPU timing information as a (user, system, idle) * tuple. On a multiprocessor system, the values returned are the @@ -907,24 +906,24 @@ psutil_virtual_mem(PyObject *self, PyObject *args) { */ static PyObject * psutil_cpu_times(PyObject *self, PyObject *args) { - float idle, kernel, user, system; + double idle, kernel, user, system; FILETIME idle_time, kernel_time, user_time; if (!GetSystemTimes(&idle_time, &kernel_time, &user_time)) return PyErr_SetFromWindowsErr(0); - idle = (float)((HI_T * idle_time.dwHighDateTime) + \ + idle = (double)((HI_T * idle_time.dwHighDateTime) + \ (LO_T * idle_time.dwLowDateTime)); - user = (float)((HI_T * user_time.dwHighDateTime) + \ + user = (double)((HI_T * user_time.dwHighDateTime) + \ (LO_T * user_time.dwLowDateTime)); - kernel = (float)((HI_T * kernel_time.dwHighDateTime) + \ + kernel = (double)((HI_T * kernel_time.dwHighDateTime) + \ (LO_T * kernel_time.dwLowDateTime)); // Kernel time includes idle time. // We return only busy kernel time subtracting idle time from // kernel time. system = (kernel - idle); - return Py_BuildValue("(fff)", user, system, idle); + return Py_BuildValue("(ddd)", user, system, idle); } @@ -938,7 +937,7 @@ psutil_per_cpu_times(PyObject *self, PyObject *args) { NTQSI_PROC NtQuerySystemInformation; HINSTANCE hNtDll; - float idle, kernel, systemt, user, interrupt, dpc; + double idle, kernel, systemt, user, interrupt, dpc; NTSTATUS status; _SYSTEM_PROCESSOR_PERFORMANCE_INFORMATION *sppi = NULL; SYSTEM_INFO si; @@ -992,15 +991,15 @@ psutil_per_cpu_times(PyObject *self, PyObject *args) { idle = user = kernel = interrupt = dpc = 0; for (i = 0; i < si.dwNumberOfProcessors; i++) { py_tuple = NULL; - user = (float)((HI_T * sppi[i].UserTime.HighPart) + + user = (double)((HI_T * sppi[i].UserTime.HighPart) + (LO_T * sppi[i].UserTime.LowPart)); - idle = (float)((HI_T * sppi[i].IdleTime.HighPart) + + idle = (double)((HI_T * sppi[i].IdleTime.HighPart) + (LO_T * sppi[i].IdleTime.LowPart)); - kernel = (float)((HI_T * sppi[i].KernelTime.HighPart) + + kernel = (double)((HI_T * sppi[i].KernelTime.HighPart) + (LO_T * sppi[i].KernelTime.LowPart)); - interrupt = (float)((HI_T * sppi[i].InterruptTime.HighPart) + + interrupt = (double)((HI_T * sppi[i].InterruptTime.HighPart) + (LO_T * sppi[i].InterruptTime.LowPart)); - dpc = (float)((HI_T * sppi[i].DpcTime.HighPart) + + dpc = (double)((HI_T * sppi[i].DpcTime.HighPart) + (LO_T * sppi[i].DpcTime.LowPart)); // kernel time includes idle time on windows @@ -2844,10 +2843,11 @@ psutil_proc_info(PyObject *self, PyObject *args) { for (i = 0; i < process->NumberOfThreads; i++) ctx_switches += process->Threads[i].ContextSwitches; - user_time = (double)process->UserTime.HighPart * 429.4967296 + \ - (double)process->UserTime.LowPart * 1e-7; - kernel_time = (double)process->KernelTime.HighPart * 429.4967296 + \ - (double)process->KernelTime.LowPart * 1e-7; + user_time = (double)process->UserTime.HighPart * HI_T + \ + (double)process->UserTime.LowPart * LO_T; + kernel_time = (double)process->KernelTime.HighPart * HI_T + \ + (double)process->KernelTime.LowPart * LO_T; + // Convert the LARGE_INTEGER union to a Unix time. // It's the best I could find by googling and borrowing code here // and there. The time returned has a precision of 1 second. From e258a673940c39675d79a29ed495abc29b98035d Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 13 Mar 2018 09:41:21 +0100 Subject: [PATCH 022/182] give CREDITS to @stswandering for #1240 --- CREDITS | 4 +++- HISTORY.rst | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CREDITS b/CREDITS index e0d748dad..deeaf45b9 100644 --- a/CREDITS +++ b/CREDITS @@ -520,4 +520,6 @@ N: Dan Vinakovsky W: https://github.com/hexaclock I: 1216 -N +N: stswandering +W: https://github.com/stswandering +I: 1243 diff --git a/HISTORY.rst b/HISTORY.rst index bf378ed80..c58c8506c 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -14,6 +14,8 @@ XXXX-XX-XX **Bug fixes** - 1216_: fix compatibility with python 2.6 on Windows (patch by Dan Vinakovsky) +- 1240_: [Windows] cpu_times() float loses accuracy in a long running system. + (patch by stswandering) 5.4.3 ===== From 006533d55316b6a9eb8004bd1b30acc3257fd15c Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 13 Mar 2018 11:46:42 +0100 Subject: [PATCH 023/182] #1245 / linux / temperatures: skip sensors if 'name' file does not exist --- HISTORY.rst | 1 + psutil/_pslinux.py | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index c58c8506c..edf1078ec 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -16,6 +16,7 @@ XXXX-XX-XX - 1216_: fix compatibility with python 2.6 on Windows (patch by Dan Vinakovsky) - 1240_: [Windows] cpu_times() float loses accuracy in a long running system. (patch by stswandering) +- 1245_: [Linux] sensors_temperatures() may fail with IOError "no such file". 5.4.3 ===== diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index b57adb34e..a3653f989 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -1141,17 +1141,18 @@ def sensors_temperatures(): for base in basenames: try: current = float(cat(base + '_input')) / 1000.0 + unit_name = cat(os.path.join(os.path.dirname(base), 'name'), + binary=False) except (IOError, OSError) as err: # A lot of things can go wrong here, so let's just skip the # whole entry. # https://github.com/giampaolo/psutil/issues/1009 # https://github.com/giampaolo/psutil/issues/1101 # https://github.com/giampaolo/psutil/issues/1129 + # https://github.com/giampaolo/psutil/issues/1245 warnings.warn("ignoring %r" % err, RuntimeWarning) continue - unit_name = cat(os.path.join(os.path.dirname(base), 'name'), - binary=False) high = cat(base + '_max', fallback=None) critical = cat(base + '_crit', fallback=None) label = cat(base + '_label', fallback='', binary=False) From a16003453fdd1082550e61f12e04e1ae14e7d9fa Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 13 Mar 2018 18:34:49 +0100 Subject: [PATCH 024/182] more detailed error msg in case of missing file when reading temperatures --- psutil/_pslinux.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index a3653f989..b241710dc 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -1140,9 +1140,10 @@ def sensors_temperatures(): for base in basenames: try: - current = float(cat(base + '_input')) / 1000.0 - unit_name = cat(os.path.join(os.path.dirname(base), 'name'), - binary=False) + path = base + '_input' + current = float(cat(path)) / 1000.0 + path = os.path.join(os.path.dirname(base), 'name') + unit_name = cat(path, binary=False) except (IOError, OSError) as err: # A lot of things can go wrong here, so let's just skip the # whole entry. @@ -1150,7 +1151,8 @@ def sensors_temperatures(): # https://github.com/giampaolo/psutil/issues/1101 # https://github.com/giampaolo/psutil/issues/1129 # https://github.com/giampaolo/psutil/issues/1245 - warnings.warn("ignoring %r" % err, RuntimeWarning) + warnings.warn("ignoring %r for file %r" % (err, path), + RuntimeWarning) continue high = cat(base + '_max', fallback=None) From d67e977cb57e77c6907d3e4c183b8ffedd9e9dcc Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 13 Mar 2018 18:38:12 +0100 Subject: [PATCH 025/182] #1222: [Linux] Process.memory_full_info() was summing both 'swap PSS' (swap proportional set size) and plain 'swap'. Not anymore. --- HISTORY.rst | 2 ++ psutil/_pslinux.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index edf1078ec..9667f85f2 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -14,6 +14,8 @@ XXXX-XX-XX **Bug fixes** - 1216_: fix compatibility with python 2.6 on Windows (patch by Dan Vinakovsky) +- 1222_: [Linux] Process.memory_full_info() was summing both "swap PSS" (swap + proportional set size) and plain "swap". Not anymore. - 1240_: [Windows] cpu_times() float loses accuracy in a long running system. (patch by stswandering) - 1245_: [Linux] sensors_temperatures() may fail with IOError "no such file". diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index b241710dc..e7f6e2c1e 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -1608,7 +1608,7 @@ def memory_full_info( self, _private_re=re.compile(br"Private.*:\s+(\d+)"), _pss_re=re.compile(br"Pss.*:\s+(\d+)"), - _swap_re=re.compile(br"Swap.*:\s+(\d+)")): + _swap_re=re.compile(br"Swap:\s+(\d+)")): basic_mem = self.memory_info() # Note: using 3 regexes is faster than reading the file # line by line. From c08ec74d44e5183dd2e399c96392bcb96d2ea971 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 13 Mar 2018 19:21:43 +0100 Subject: [PATCH 026/182] #1222 / linux / memory_full_info: we were erroneously adding "Pss:" and "SwapPss" --- HISTORY.rst | 4 ++-- psutil/_pslinux.py | 7 ++++--- psutil/tests/test_linux.py | 40 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 46 insertions(+), 5 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 9667f85f2..bb0949802 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -14,8 +14,8 @@ XXXX-XX-XX **Bug fixes** - 1216_: fix compatibility with python 2.6 on Windows (patch by Dan Vinakovsky) -- 1222_: [Linux] Process.memory_full_info() was summing both "swap PSS" (swap - proportional set size) and plain "swap". Not anymore. +- 1222_: [Linux] Process.memory_full_info() was erroneously summing "Swap:" and + "SwapPss:". Same for "Pss:" and "SwapPss". Not anymore. - 1240_: [Windows] cpu_times() float loses accuracy in a long running system. (patch by stswandering) - 1245_: [Linux] sensors_temperatures() may fail with IOError "no such file". diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index e7f6e2c1e..c8c10ec9b 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -1606,9 +1606,10 @@ def memory_info(self): @wrap_exceptions def memory_full_info( self, - _private_re=re.compile(br"Private.*:\s+(\d+)"), - _pss_re=re.compile(br"Pss.*:\s+(\d+)"), - _swap_re=re.compile(br"Swap:\s+(\d+)")): + # Gets Private_Clean, Private_Dirty, Private_Hugetlb. + _private_re=re.compile(br"\nPrivate.*:\s+(\d+)"), + _pss_re=re.compile(br"\nPss\:\s+(\d+)"), + _swap_re=re.compile(br"\nSwap\:\s+(\d+)")): basic_mem = self.memory_info() # Note: using 3 regexes is faster than reading the file # line by line. diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index 6ba17b254..139b32abe 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -1482,6 +1482,46 @@ def test_memory_full_info(self): self.assertAlmostEqual( mem.swap, sum([x.swap for x in maps]), delta=4096) + def test_memory_full_info_mocked(self): + # See: https://github.com/giampaolo/psutil/issues/1222 + def open_mock(name, *args, **kwargs): + if name == "/proc/%s/smaps" % os.getpid(): + return io.BytesIO(textwrap.dedent("""\ + fffff0 r-xp 00000000 00:00 0 [vsyscall] + Size: 1 kB + Rss: 2 kB + Pss: 3 kB + Shared_Clean: 4 kB + Shared_Dirty: 5 kB + Private_Clean: 6 kB + Private_Dirty: 7 kB + Referenced: 8 kB + Anonymous: 9 kB + LazyFree: 10 kB + AnonHugePages: 11 kB + ShmemPmdMapped: 12 kB + Shared_Hugetlb: 13 kB + Private_Hugetlb: 14 kB + Swap: 15 kB + SwapPss: 16 kB + KernelPageSize: 17 kB + MMUPageSize: 18 kB + Locked: 19 kB + VmFlags: rd ex + """).encode()) + else: + return orig_open(name, *args, **kwargs) + + orig_open = open + patch_point = 'builtins.open' if PY3 else '__builtin__.open' + with mock.patch(patch_point, create=True, side_effect=open_mock) as m: + p = psutil.Process() + mem = p.memory_full_info() + assert m.called + self.assertEqual(mem.uss, (6 + 7 + 14) * 1024) + self.assertEqual(mem.pss, 3 * 1024) + self.assertEqual(mem.swap, 15 * 1024) + # On PYPY file descriptors are not closed fast enough. @unittest.skipIf(PYPY, "unreliable on PYPY") def test_open_files_mode(self): From 7b0e1c43e7d470bc4c5ef0ee734792b57f25ee70 Mon Sep 17 00:00:00 2001 From: wiggin15 Date: Thu, 15 Mar 2018 00:29:22 +0200 Subject: [PATCH 027/182] Ignore negative deltas in cpu times when calculating percentages (#1210) (#1214) --- psutil/__init__.py | 66 ++++++++++++++++++++------------------ psutil/tests/test_linux.py | 51 +++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+), 31 deletions(-) diff --git a/psutil/__init__.py b/psutil/__init__.py index a11988ba2..1b0f2c045 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -1655,6 +1655,27 @@ def _cpu_busy_time(times): return busy +def _cpu_times_deltas(t1, t2): + assert t1._fields == t2._fields, (t1, t2) + field_deltas = [] + for field in _psplatform.scputimes._fields: + field_delta = getattr(t2, field) - getattr(t1, field) + # CPU times are always supposed to increase over time + # or at least remain the same and that's because time + # cannot go backwards. + # Surprisingly sometimes this might not be the case (at + # least on Windows and Linux), see: + # https://github.com/giampaolo/psutil/issues/392 + # https://github.com/giampaolo/psutil/issues/645 + # https://github.com/giampaolo/psutil/issues/1210 + # Trim negative deltas to zero to ignore decreasing fields. + # top does the same. Reference: + # https://gitlab.com/procps-ng/procps/blob/v3.3.12/top/top.c#L5063 + field_delta = max(0, field_delta) + field_deltas.append(field_delta) + return _psplatform.scputimes(*field_deltas) + + def cpu_percent(interval=None, percpu=False): """Return a float representing the current system-wide CPU utilization as a percentage. @@ -1697,18 +1718,11 @@ def cpu_percent(interval=None, percpu=False): raise ValueError("interval is not positive (got %r)" % interval) def calculate(t1, t2): - t1_all = _cpu_tot_time(t1) - t1_busy = _cpu_busy_time(t1) + times_delta = _cpu_times_deltas(t1, t2) - t2_all = _cpu_tot_time(t2) - t2_busy = _cpu_busy_time(t2) + all_delta = _cpu_tot_time(times_delta) + busy_delta = _cpu_busy_time(times_delta) - # this usually indicates a float precision issue - if t2_busy <= t1_busy: - return 0.0 - - busy_delta = t2_busy - t1_busy - all_delta = t2_all - t1_all try: busy_perc = (busy_delta / all_delta) * 100 except ZeroDivisionError: @@ -1777,28 +1791,18 @@ def cpu_times_percent(interval=None, percpu=False): def calculate(t1, t2): nums = [] - all_delta = _cpu_tot_time(t2) - _cpu_tot_time(t1) - for field in t1._fields: - field_delta = getattr(t2, field) - getattr(t1, field) - try: - field_perc = (100 * field_delta) / all_delta - except ZeroDivisionError: - field_perc = 0.0 + times_delta = _cpu_times_deltas(t1, t2) + all_delta = _cpu_tot_time(times_delta) + # "scale" is the value to multiply each delta with to get percentages. + # We use "max" to avoid division by zero (if all_delta is 0, then all + # fields are 0 so percentages will be 0 too. all_delta cannot be a + # fraction because cpu times are integers) + scale = 100.0 / max(1, all_delta) + for field_delta in times_delta: + field_perc = field_delta * scale field_perc = round(field_perc, 1) - # CPU times are always supposed to increase over time - # or at least remain the same and that's because time - # cannot go backwards. - # Surprisingly sometimes this might not be the case (at - # least on Windows and Linux), see: - # https://github.com/giampaolo/psutil/issues/392 - # https://github.com/giampaolo/psutil/issues/645 - # I really don't know what to do about that except - # forcing the value to 0 or 100. - if field_perc > 100.0: - field_perc = 100.0 - # `<=` because `-0.0 == 0.0` evaluates to True - elif field_perc <= 0.0: - field_perc = 0.0 + # make sure we don't return negative values or values over 100% + field_perc = min(max(0.0, field_perc), 100.0) nums.append(field_perc) return _psplatform.scputimes(*nums) diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index 139b32abe..090634165 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -1090,6 +1090,57 @@ def open_mock(name, *args, **kwargs): self.assertEqual(psutil.PROCFS_PATH, '/proc') + def test_cpu_steal_decrease(self): + # Test cumulative cpu stats decrease. We should ignore this. + # See issue #1210. + + def open_mock(name, *args, **kwargs): + if name == "/proc/stat": + return io.BytesIO(textwrap.dedent("""\ + cpu 0 0 0 0 0 0 0 1 0 0 + cpu0 0 0 0 0 0 0 0 1 0 0 + cpu1 0 0 0 0 0 0 0 1 0 0 + """).encode()) + return orig_open(name, *args, **kwargs) + + orig_open = open + patch_point = 'builtins.open' if PY3 else '__builtin__.open' + + with mock.patch(patch_point, create=True, side_effect=open_mock) as m: + # first call to "percent" functions should read the new stat file + # and compare to the "real" file read at import time - so the + # values are meaningless + psutil.cpu_percent() + assert m.called + psutil.cpu_percent(percpu=True) + psutil.cpu_times_percent() + psutil.cpu_times_percent(percpu=True) + + def open_mock(name, *args, **kwargs): + if name == "/proc/stat": + return io.BytesIO(textwrap.dedent("""\ + cpu 1 0 0 0 0 0 0 0 0 0 + cpu0 1 0 0 0 0 0 0 0 0 0 + cpu1 1 0 0 0 0 0 0 0 0 0 + """).encode()) + return orig_open(name, *args, **kwargs) + + with mock.patch(patch_point, create=True, side_effect=open_mock) as m: + # Increase "user" while steal goes "backwards" to zero. + cpu_percent = psutil.cpu_percent() + assert m.called + cpu_percent_percpu = psutil.cpu_percent(percpu=True) + cpu_times_percent = psutil.cpu_times_percent() + cpu_times_percent_percpu = psutil.cpu_times_percent(percpu=True) + self.assertNotEqual(cpu_percent, 0) + self.assertNotEqual(sum(cpu_percent_percpu), 0) + self.assertNotEqual(sum(cpu_times_percent), 0) + self.assertNotEqual(sum(cpu_times_percent), 100.0) + self.assertNotEqual(sum(map(sum, cpu_times_percent_percpu)), 0) + self.assertNotEqual(sum(map(sum, cpu_times_percent_percpu)), 100.0) + self.assertEqual(cpu_times_percent.steal, 0) + self.assertNotEqual(cpu_times_percent.user, 0) + def test_boot_time_mocked(self): with mock.patch('psutil._pslinux.open', create=True) as m: self.assertRaises( From c088fb59a2feb92104caa7752344fccca341b42c Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 14 Mar 2018 23:37:07 +0100 Subject: [PATCH 028/182] #1210, #1214: update README and give CREDITs to @wiggin15 --- CREDITS | 2 +- HISTORY.rst | 3 +++ README.rst | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CREDITS b/CREDITS index deeaf45b9..1ae6622d8 100644 --- a/CREDITS +++ b/CREDITS @@ -57,7 +57,7 @@ W: http://www.jayloden.com N: Arnon Yaari (wiggin15) W: https://github.com/wiggin15 -I: 517, 607, 610, 1131, 1123, 1130, 1154, 1164, 1174, 1177 +I: 517, 607, 610, 1131, 1123, 1130, 1154, 1164, 1174, 1177, 1210, 1214 N: Jeff Tang W: https://github.com/mrjefftang diff --git a/HISTORY.rst b/HISTORY.rst index bb0949802..49e9ed4c2 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -13,6 +13,9 @@ XXXX-XX-XX **Bug fixes** +- 1210_: [Linux] cpu_percent() steal time may remain stuch at 100% due to Linux + erroneously reporting a decreased steal time between calls. (patch by Arnon + Yaari) - 1216_: fix compatibility with python 2.6 on Windows (patch by Dan Vinakovsky) - 1222_: [Linux] Process.memory_full_info() was erroneously summing "Swap:" and "SwapPss:". Same for "Pss:" and "SwapPss". Not anymore. diff --git a/README.rst b/README.rst index 04e4bd6bd..73205270c 100644 --- a/README.rst +++ b/README.rst @@ -86,7 +86,7 @@ Projects using psutil At the time of writing psutil has roughly `2.9 milion downloads `__ per month and there are over -`6000 open source projects `__ +`7000 open source projects `__ on github which depend from psutil. Here's some I find particularly interesting: From 6276764eb98eccaef87053dba843694c9232a98c Mon Sep 17 00:00:00 2001 From: Georg Sauthoff Date: Sat, 17 Mar 2018 10:31:08 +0100 Subject: [PATCH 029/182] Solaris 10 Fixes (#1248) * Fix nice() for realtime processes under Solaris 10 fixes #1194 * Use psinfo as fallback [gu]id source on Solaris 10 fixes #1193 * Fix double free * Match ssize_t return type of read functions * Fix undefined behavior with respect to strict aliasing rules and fix some warnings For example, under strict aliasing rules of the C standard, casting a char pointer to a struct pointer and accessing the character array through that struct pointer yields undefined behavior. * Update HISTORY with Solaris notes --- HISTORY.rst | 7 ++ psutil/_pssunos.py | 50 ++++++------ psutil/_psutil_common.c | 4 +- psutil/_psutil_common.h | 9 ++- psutil/_psutil_sunos.c | 174 ++++++++++++++++++++-------------------- 5 files changed, 127 insertions(+), 117 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 49e9ed4c2..fbf5ac004 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -22,6 +22,13 @@ XXXX-XX-XX - 1240_: [Windows] cpu_times() float loses accuracy in a long running system. (patch by stswandering) - 1245_: [Linux] sensors_temperatures() may fail with IOError "no such file". +- 1193_: [SunOS] Return uid/gid from /proc/pid/psinfo if there aren't + enough permissions for /proc/pid/cred +- 1194_: [SunOS] Return nice value from psinfo as getpriority() doesn't + support real-time processes +- 1194_: [SunOS] Fix double free in psutil_proc_cpu_num() +- 1194_: [SunOS] Fix undefined behavior related to strict-aliasing rules + and warnings 5.4.3 ===== diff --git a/psutil/_pssunos.py b/psutil/_pssunos.py index 5471d5aa4..35b4b092b 100644 --- a/psutil/_pssunos.py +++ b/psutil/_pssunos.py @@ -79,7 +79,11 @@ nice=4, num_threads=5, status=6, - ttynr=7) + ttynr=7, + uid=8, + euid=9, + gid=10, + egid=11) # ===================================================================== @@ -394,7 +398,10 @@ def _proc_basic_info(self): @memoize_when_activated def _proc_cred(self): - return cext.proc_cred(self.pid, self._procfs_path) + @wrap_exceptions + def proc_cred(self): + return cext.proc_cred(self.pid, self._procfs_path) + return proc_cred(self) @wrap_exceptions def name(self): @@ -432,27 +439,10 @@ def num_threads(self): @wrap_exceptions def nice_get(self): - # Note #1: for some reason getpriority(3) return ESRCH (no such - # process) for certain low-pid processes, no matter what (even - # as root). - # The process actually exists though, as it has a name, - # creation time, etc. - # The best thing we can do here appears to be raising AD. - # Note: tested on Solaris 11; on Open Solaris 5 everything is - # fine. - # - # Note #2: we also can get niceness from /proc/pid/psinfo - # but it's wrong, see: - # https://github.com/giampaolo/psutil/issues/1082 - try: - return cext_posix.getpriority(self.pid) - except EnvironmentError as err: - # 48 is 'operation not supported' but errno does not expose - # it. It occurs for low system pids. - if err.errno in (errno.ENOENT, errno.ESRCH, 48): - if pid_exists(self.pid): - raise AccessDenied(self.pid, self._name) - raise + # Note #1: getpriority(3) doesn't work for realtime processes. + # Psinfo is what ps uses, see: + # https://github.com/giampaolo/psutil/issues/1194 + return self._proc_basic_info()[proc_info_map['nice']] @wrap_exceptions def nice_set(self, value): @@ -471,12 +461,22 @@ def ppid(self): @wrap_exceptions def uids(self): - real, effective, saved, _, _, _ = self._proc_cred() + try: + real, effective, saved, _, _, _ = self._proc_cred() + except AccessDenied: + real = self._proc_basic_info()[proc_info_map['uid']] + effective = self._proc_basic_info()[proc_info_map['euid']] + saved = None return _common.puids(real, effective, saved) @wrap_exceptions def gids(self): - _, _, _, real, effective, saved = self._proc_cred() + try: + _, _, _, real, effective, saved = self._proc_cred() + except AccessDenied: + real = self._proc_basic_info()[proc_info_map['gid']] + effective = self._proc_basic_info()[proc_info_map['egid']] + saved = None return _common.puids(real, effective, saved) @wrap_exceptions diff --git a/psutil/_psutil_common.c b/psutil/_psutil_common.c index 908dbf14a..e08f011c2 100644 --- a/psutil/_psutil_common.c +++ b/psutil/_psutil_common.c @@ -40,7 +40,7 @@ PyUnicode_DecodeFSDefaultAndSize(char *s, Py_ssize_t size) { * If msg != "" the exception message will change in accordance. */ PyObject * -NoSuchProcess(char *msg) { +NoSuchProcess(const char *msg) { PyObject *exc; exc = PyObject_CallFunction( PyExc_OSError, "(is)", ESRCH, strlen(msg) ? msg : strerror(ESRCH)); @@ -55,7 +55,7 @@ NoSuchProcess(char *msg) { * If msg != "" the exception message will change in accordance. */ PyObject * -AccessDenied(char *msg) { +AccessDenied(const char *msg) { PyObject *exc; exc = PyObject_CallFunction( PyExc_OSError, "(is)", EACCES, strlen(msg) ? msg : strerror(EACCES)); diff --git a/psutil/_psutil_common.h b/psutil/_psutil_common.h index 3db3f5ede..e107166a1 100644 --- a/psutil/_psutil_common.h +++ b/psutil/_psutil_common.h @@ -4,6 +4,9 @@ * found in the LICENSE file. */ +#ifndef PSUTIL_PSUTIL_COMMON_H +#define PSUTIL_PSUTIL_COMMON_H + #include extern int PSUTIL_TESTING; @@ -17,9 +20,11 @@ PyObject* PyUnicode_DecodeFSDefault(char *s); PyObject* PyUnicode_DecodeFSDefaultAndSize(char *s, Py_ssize_t size); #endif -PyObject* AccessDenied(char *msg); -PyObject* NoSuchProcess(char *msg); +PyObject* AccessDenied(const char *msg); +PyObject* NoSuchProcess(const char *msg); PyObject* psutil_set_testing(PyObject *self, PyObject *args); void psutil_debug(const char* format, ...); void psutil_setup(void); + +#endif // PSUTIL_PSUTIL_COMMON_H diff --git a/psutil/_psutil_sunos.c b/psutil/_psutil_sunos.c index 4d38a3423..99069f56a 100644 --- a/psutil/_psutil_sunos.c +++ b/psutil/_psutil_sunos.c @@ -46,6 +46,7 @@ #include #include #include +#include // fabs() #include "_psutil_common.h" #include "_psutil_posix.h" @@ -58,10 +59,10 @@ /* * Read a file content and fills a C structure with it. */ -int +static int psutil_file_to_struct(char *path, void *fstruct, size_t size) { int fd; - size_t nbytes; + ssize_t nbytes; fd = open(path, O_RDONLY); if (fd == -1) { PyErr_SetFromErrnoWithFilename(PyExc_OSError, path); @@ -73,7 +74,7 @@ psutil_file_to_struct(char *path, void *fstruct, size_t size) { PyErr_SetFromErrno(PyExc_OSError); return 0; } - if (nbytes != size) { + if (nbytes != (ssize_t) size) { close(fd); PyErr_SetString( PyExc_RuntimeError, "read() file structure size mismatch"); @@ -102,17 +103,19 @@ psutil_proc_basic_info(PyObject *self, PyObject *args) { if (! psutil_file_to_struct(path, (void *)&info, sizeof(info))) return NULL; return Py_BuildValue( - "ikkdiiik", + "ikkdiiikiiii", info.pr_ppid, // parent pid info.pr_rssize, // rss info.pr_size, // vms PSUTIL_TV2DOUBLE(info.pr_start), // create time - // XXX - niceness is wrong (20 instead of 0), see: - // https://github.com/giampaolo/psutil/issues/1082 info.pr_lwp.pr_nice, // nice info.pr_nlwp, // no. of threads info.pr_lwp.pr_state, // status code - info.pr_ttydev // tty nr + info.pr_ttydev, // tty nr + (int)info.pr_uid, // real user id + (int)info.pr_euid, // effective user id + (int)info.pr_gid, // real group id + (int)info.pr_egid // effective group id ); } @@ -348,13 +351,11 @@ psutil_proc_cpu_num(PyObject *self, PyObject *args) { int pid; char path[1000]; struct prheader header; - struct lwpsinfo *lwp; - char *lpsinfo = NULL; - char *ptr = NULL; + struct lwpsinfo *lwp = NULL; int nent; int size; int proc_num; - size_t nbytes; + ssize_t nbytes; const char *procfs_path; if (! PyArg_ParseTuple(args, "is", &pid, &procfs_path)) @@ -382,14 +383,14 @@ psutil_proc_cpu_num(PyObject *self, PyObject *args) { // malloc nent = header.pr_nent; size = header.pr_entsize * nent; - ptr = lpsinfo = malloc(size); - if (lpsinfo == NULL) { + lwp = malloc(size); + if (lwp == NULL) { PyErr_NoMemory(); goto error; } // read the rest - nbytes = pread(fd, lpsinfo, size, sizeof(header)); + nbytes = pread(fd, lwp, size, sizeof(header)); if (nbytes == -1) { PyErr_SetFromErrno(PyExc_OSError); goto error; @@ -401,20 +402,15 @@ psutil_proc_cpu_num(PyObject *self, PyObject *args) { } // done - lwp = (lwpsinfo_t *)ptr; proc_num = lwp->pr_onpro; close(fd); - free(ptr); - free(lpsinfo); + free(lwp); return Py_BuildValue("i", proc_num); error: if (fd != -1) close(fd); - if (ptr != NULL) - free(ptr); - if (lpsinfo != NULL) - free(lpsinfo); + free(lwp); return NULL; } @@ -853,7 +849,7 @@ psutil_proc_memory_maps(PyObject *self, PyObject *args) { int fd = -1; char path[1000]; char perms[10]; - char *name; + const char *name; struct stat st; pstatus_t status; @@ -1115,11 +1111,11 @@ static PyObject * psutil_net_connections(PyObject *self, PyObject *args) { long pid; int sd = 0; - mib2_tcpConnEntry_t *tp = NULL; - mib2_udpEntry_t *ude; + mib2_tcpConnEntry_t tp; + mib2_udpEntry_t ude; #if defined(AF_INET6) - mib2_tcp6ConnEntry_t *tp6; - mib2_udp6Entry_t *ude6; + mib2_tcp6ConnEntry_t tp6; + mib2_udp6Entry_t ude6; #endif char buf[512]; int i, flags, getcode, num_ent, state; @@ -1128,10 +1124,10 @@ psutil_net_connections(PyObject *self, PyObject *args) { int processed_pid; int databuf_init = 0; struct strbuf ctlbuf, databuf; - struct T_optmgmt_req *tor = (struct T_optmgmt_req *)buf; - struct T_optmgmt_ack *toa = (struct T_optmgmt_ack *)buf; - struct T_error_ack *tea = (struct T_error_ack *)buf; - struct opthdr *mibhdr; + struct T_optmgmt_req tor = {0}; + struct T_optmgmt_ack toa = {0}; + struct T_error_ack tea = {0}; + struct opthdr mibhdr = {0}; PyObject *py_retlist = PyList_New(0); PyObject *py_tuple = NULL; @@ -1164,22 +1160,23 @@ psutil_net_connections(PyObject *self, PyObject *args) { // which copied and pasted it from netstat source code, mibget() // function. Also see: // http://stackoverflow.com/questions/8723598/ - tor->PRIM_type = T_SVR4_OPTMGMT_REQ; - tor->OPT_offset = sizeof (struct T_optmgmt_req); - tor->OPT_length = sizeof (struct opthdr); - tor->MGMT_flags = T_CURRENT; - mibhdr = (struct opthdr *)&tor[1]; - mibhdr->level = MIB2_IP; - mibhdr->name = 0; + tor.PRIM_type = T_SVR4_OPTMGMT_REQ; + tor.OPT_offset = sizeof (struct T_optmgmt_req); + tor.OPT_length = sizeof (struct opthdr); + tor.MGMT_flags = T_CURRENT; + mibhdr.level = MIB2_IP; + mibhdr.name = 0; #ifdef NEW_MIB_COMPLIANT - mibhdr->len = 1; + mibhdr.len = 1; #else - mibhdr->len = 0; + mibhdr.len = 0; #endif + memcpy(buf, &tor, sizeof tor); + memcpy(buf + sizeof tor, &mibhdr, sizeof mibhdr); ctlbuf.buf = buf; - ctlbuf.len = tor->OPT_offset + tor->OPT_length; + ctlbuf.len = tor.OPT_offset + tor.OPT_length; flags = 0; // request to be sent in non-priority if (putmsg(sd, &ctlbuf, (struct strbuf *)0, flags) == -1) { @@ -1187,37 +1184,38 @@ psutil_net_connections(PyObject *self, PyObject *args) { goto error; } - mibhdr = (struct opthdr *)&toa[1]; ctlbuf.maxlen = sizeof (buf); for (;;) { flags = 0; getcode = getmsg(sd, &ctlbuf, (struct strbuf *)0, &flags); + memcpy(&toa, buf, sizeof toa); + memcpy(&tea, buf, sizeof tea); if (getcode != MOREDATA || - ctlbuf.len < sizeof (struct T_optmgmt_ack) || - toa->PRIM_type != T_OPTMGMT_ACK || - toa->MGMT_flags != T_SUCCESS) + ctlbuf.len < (int)sizeof (struct T_optmgmt_ack) || + toa.PRIM_type != T_OPTMGMT_ACK || + toa.MGMT_flags != T_SUCCESS) { break; } - if (ctlbuf.len >= sizeof (struct T_error_ack) && - tea->PRIM_type == T_ERROR_ACK) + if (ctlbuf.len >= (int)sizeof (struct T_error_ack) && + tea.PRIM_type == T_ERROR_ACK) { PyErr_SetString(PyExc_RuntimeError, "ERROR_ACK"); goto error; } if (getcode == 0 && - ctlbuf.len >= sizeof (struct T_optmgmt_ack) && - toa->PRIM_type == T_OPTMGMT_ACK && - toa->MGMT_flags == T_SUCCESS) + ctlbuf.len >= (int)sizeof (struct T_optmgmt_ack) && + toa.PRIM_type == T_OPTMGMT_ACK && + toa.MGMT_flags == T_SUCCESS) { PyErr_SetString(PyExc_RuntimeError, "ERROR_T_OPTMGMT_ACK"); goto error; } - databuf.maxlen = mibhdr->len; + databuf.maxlen = mibhdr.len; databuf.len = 0; - databuf.buf = (char *)malloc((int)mibhdr->len); + databuf.buf = (char *)malloc((int)mibhdr.len); if (!databuf.buf) { PyErr_NoMemory(); goto error; @@ -1232,22 +1230,22 @@ psutil_net_connections(PyObject *self, PyObject *args) { } // TCPv4 - if (mibhdr->level == MIB2_TCP && mibhdr->name == MIB2_TCP_13) { - tp = (mib2_tcpConnEntry_t *)databuf.buf; - num_ent = mibhdr->len / sizeof(mib2_tcpConnEntry_t); - for (i = 0; i < num_ent; i++, tp++) { + if (mibhdr.level == MIB2_TCP && mibhdr.name == MIB2_TCP_13) { + num_ent = mibhdr.len / sizeof(mib2_tcpConnEntry_t); + for (i = 0; i < num_ent; i++) { + memcpy(&tp, databuf.buf + i * sizeof tp, sizeof tp); #ifdef NEW_MIB_COMPLIANT - processed_pid = tp->tcpConnCreationProcess; + processed_pid = tp.tcpConnCreationProcess; #else processed_pid = 0; #endif if (pid != -1 && processed_pid != pid) continue; // construct local/remote addresses - inet_ntop(AF_INET, &tp->tcpConnLocalAddress, lip, sizeof(lip)); - inet_ntop(AF_INET, &tp->tcpConnRemAddress, rip, sizeof(rip)); - lport = tp->tcpConnLocalPort; - rport = tp->tcpConnRemPort; + inet_ntop(AF_INET, &tp.tcpConnLocalAddress, lip, sizeof(lip)); + inet_ntop(AF_INET, &tp.tcpConnRemAddress, rip, sizeof(rip)); + lport = tp.tcpConnLocalPort; + rport = tp.tcpConnRemPort; // contruct python tuple/list py_laddr = Py_BuildValue("(si)", lip, lport); @@ -1260,7 +1258,7 @@ psutil_net_connections(PyObject *self, PyObject *args) { } if (!py_raddr) goto error; - state = tp->tcpConnEntryInfo.ce_state; + state = tp.tcpConnEntryInfo.ce_state; // add item py_tuple = Py_BuildValue("(iiiNNiI)", -1, AF_INET, SOCK_STREAM, @@ -1275,24 +1273,24 @@ psutil_net_connections(PyObject *self, PyObject *args) { } #if defined(AF_INET6) // TCPv6 - else if (mibhdr->level == MIB2_TCP6 && mibhdr->name == MIB2_TCP6_CONN) + else if (mibhdr.level == MIB2_TCP6 && mibhdr.name == MIB2_TCP6_CONN) { - tp6 = (mib2_tcp6ConnEntry_t *)databuf.buf; - num_ent = mibhdr->len / sizeof(mib2_tcp6ConnEntry_t); + num_ent = mibhdr.len / sizeof(mib2_tcp6ConnEntry_t); - for (i = 0; i < num_ent; i++, tp6++) { + for (i = 0; i < num_ent; i++) { + memcpy(&tp6, databuf.buf + i * sizeof tp6, sizeof tp6); #ifdef NEW_MIB_COMPLIANT - processed_pid = tp6->tcp6ConnCreationProcess; + processed_pid = tp6.tcp6ConnCreationProcess; #else processed_pid = 0; #endif if (pid != -1 && processed_pid != pid) continue; // construct local/remote addresses - inet_ntop(AF_INET6, &tp6->tcp6ConnLocalAddress, lip, sizeof(lip)); - inet_ntop(AF_INET6, &tp6->tcp6ConnRemAddress, rip, sizeof(rip)); - lport = tp6->tcp6ConnLocalPort; - rport = tp6->tcp6ConnRemPort; + inet_ntop(AF_INET6, &tp6.tcp6ConnLocalAddress, lip, sizeof(lip)); + inet_ntop(AF_INET6, &tp6.tcp6ConnRemAddress, rip, sizeof(rip)); + lport = tp6.tcp6ConnLocalPort; + rport = tp6.tcp6ConnRemPort; // contruct python tuple/list py_laddr = Py_BuildValue("(si)", lip, lport); @@ -1304,7 +1302,7 @@ psutil_net_connections(PyObject *self, PyObject *args) { py_raddr = Py_BuildValue("()"); if (!py_raddr) goto error; - state = tp6->tcp6ConnEntryInfo.ce_state; + state = tp6.tcp6ConnEntryInfo.ce_state; // add item py_tuple = Py_BuildValue("(iiiNNiI)", -1, AF_INET6, SOCK_STREAM, @@ -1318,13 +1316,13 @@ psutil_net_connections(PyObject *self, PyObject *args) { } #endif // UDPv4 - else if (mibhdr->level == MIB2_UDP || mibhdr->level == MIB2_UDP_ENTRY) { - ude = (mib2_udpEntry_t *)databuf.buf; - num_ent = mibhdr->len / sizeof(mib2_udpEntry_t); - assert(num_ent * sizeof(mib2_udpEntry_t) == mibhdr->len); - for (i = 0; i < num_ent; i++, ude++) { + else if (mibhdr.level == MIB2_UDP || mibhdr.level == MIB2_UDP_ENTRY) { + num_ent = mibhdr.len / sizeof(mib2_udpEntry_t); + assert(num_ent * sizeof(mib2_udpEntry_t) == mibhdr.len); + for (i = 0; i < num_ent; i++) { + memcpy(&ude, databuf.buf + i * sizeof ude, sizeof ude); #ifdef NEW_MIB_COMPLIANT - processed_pid = ude->udpCreationProcess; + processed_pid = ude.udpCreationProcess; #else processed_pid = 0; #endif @@ -1337,8 +1335,8 @@ psutil_net_connections(PyObject *self, PyObject *args) { // to do other than skipping. if (processed_pid > 131072) continue; - inet_ntop(AF_INET, &ude->udpLocalAddress, lip, sizeof(lip)); - lport = ude->udpLocalPort; + inet_ntop(AF_INET, &ude.udpLocalAddress, lip, sizeof(lip)); + lport = ude.udpLocalPort; py_laddr = Py_BuildValue("(si)", lip, lport); if (!py_laddr) goto error; @@ -1357,21 +1355,21 @@ psutil_net_connections(PyObject *self, PyObject *args) { } #if defined(AF_INET6) // UDPv6 - else if (mibhdr->level == MIB2_UDP6 || - mibhdr->level == MIB2_UDP6_ENTRY) + else if (mibhdr.level == MIB2_UDP6 || + mibhdr.level == MIB2_UDP6_ENTRY) { - ude6 = (mib2_udp6Entry_t *)databuf.buf; - num_ent = mibhdr->len / sizeof(mib2_udp6Entry_t); - for (i = 0; i < num_ent; i++, ude6++) { + num_ent = mibhdr.len / sizeof(mib2_udp6Entry_t); + for (i = 0; i < num_ent; i++) { + memcpy(&ude6, databuf.buf + i * sizeof ude6, sizeof ude6); #ifdef NEW_MIB_COMPLIANT - processed_pid = ude6->udp6CreationProcess; + processed_pid = ude6.udp6CreationProcess; #else processed_pid = 0; #endif if (pid != -1 && processed_pid != pid) continue; - inet_ntop(AF_INET6, &ude6->udp6LocalAddress, lip, sizeof(lip)); - lport = ude6->udp6LocalPort; + inet_ntop(AF_INET6, &ude6.udp6LocalAddress, lip, sizeof(lip)); + lport = ude6.udp6LocalPort; py_laddr = Py_BuildValue("(si)", lip, lport); if (!py_laddr) goto error; @@ -1421,7 +1419,7 @@ psutil_boot_time(PyObject *self, PyObject *args) { } } endutxent(); - if (boot_time == 0.0) { + if (fabs(boot_time) < 0.000001) { /* could not find BOOT_TIME in getutxent loop */ PyErr_SetString(PyExc_RuntimeError, "can't determine boot time"); return NULL; From 12ef896b2c175d6d9b060b95fdb06e9b915512b4 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 17 Mar 2018 10:31:28 +0100 Subject: [PATCH 030/182] fyx typo --- HISTORY.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index 49e9ed4c2..848833d12 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -13,7 +13,7 @@ XXXX-XX-XX **Bug fixes** -- 1210_: [Linux] cpu_percent() steal time may remain stuch at 100% due to Linux +- 1210_: [Linux] cpu_percent() steal time may remain stuck at 100% due to Linux erroneously reporting a decreased steal time between calls. (patch by Arnon Yaari) - 1216_: fix compatibility with python 2.6 on Windows (patch by Dan Vinakovsky) From 05d179e7125dc61409b2a8b4a33d4e3be763c2a7 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 17 Mar 2018 10:35:49 +0100 Subject: [PATCH 031/182] #1193, #1194: give credits to @gsauthof --- CREDITS | 4 ++++ HISTORY.rst | 17 +++++++++-------- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/CREDITS b/CREDITS index 1ae6622d8..f0327a7b3 100644 --- a/CREDITS +++ b/CREDITS @@ -523,3 +523,7 @@ I: 1216 N: stswandering W: https://github.com/stswandering I: 1243 + +N: Georg Sauthoff +W: https://github.com/gsauthof +I: 1193, 1194 diff --git a/HISTORY.rst b/HISTORY.rst index 27f1e9300..ddcc5a63f 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -7,12 +7,20 @@ XXXX-XX-XX **Enhancements** -- 1120_: [SunOS] cmdline() could be truncated at the 15th character when +- 694_: [SunOS] cmdline() could be truncated at the 15th character when reading it from /proc. An extra effort is made by reading it from process address space first. **Bug fixes** +- 1193_: [SunOS] Return uid/gid from /proc/pid/psinfo if there aren't + enough permissions for /proc/pid/cred. (patch by Georg Sauthoff) +- 1194_: [SunOS] Return nice value from psinfo as getpriority() doesn't + support real-time processes. (patch by Georg Sauthoff) +- 1194_: [SunOS] Fix double free in psutil_proc_cpu_num(). (patch by Georg + Sauthoff) +- 1194_: [SunOS] Fix undefined behavior related to strict-aliasing rules + and warnings. (patch by Georg Sauthoff) - 1210_: [Linux] cpu_percent() steal time may remain stuck at 100% due to Linux erroneously reporting a decreased steal time between calls. (patch by Arnon Yaari) @@ -22,13 +30,6 @@ XXXX-XX-XX - 1240_: [Windows] cpu_times() float loses accuracy in a long running system. (patch by stswandering) - 1245_: [Linux] sensors_temperatures() may fail with IOError "no such file". -- 1193_: [SunOS] Return uid/gid from /proc/pid/psinfo if there aren't - enough permissions for /proc/pid/cred -- 1194_: [SunOS] Return nice value from psinfo as getpriority() doesn't - support real-time processes -- 1194_: [SunOS] Fix double free in psutil_proc_cpu_num() -- 1194_: [SunOS] Fix undefined behavior related to strict-aliasing rules - and warnings 5.4.3 ===== From 2430e050ea2bd89d8714c823860880eccede573f Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 17 Mar 2018 10:37:52 +0100 Subject: [PATCH 032/182] update HISTORY --- HISTORY.rst | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index ddcc5a63f..078ebff28 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -5,14 +5,11 @@ XXXX-XX-XX -**Enhancements** +**Bug fixes** - 694_: [SunOS] cmdline() could be truncated at the 15th character when reading it from /proc. An extra effort is made by reading it from process - address space first. - -**Bug fixes** - + address space first. (patch by Georg Sauthoff) - 1193_: [SunOS] Return uid/gid from /proc/pid/psinfo if there aren't enough permissions for /proc/pid/cred. (patch by Georg Sauthoff) - 1194_: [SunOS] Return nice value from psinfo as getpriority() doesn't From fd78d32681c78dd81742dae6bb7b4b0da3d18010 Mon Sep 17 00:00:00 2001 From: maxime mouial Date: Sat, 17 Mar 2018 05:43:48 -0400 Subject: [PATCH 033/182] Adding 'slab' info to virtual_memory on linux (#1239) There are 1970 results on stackoverflow for "slab". Also it is being requested by another user https://github.com/giampaolo/psutil/issues/1202. As such I'm going to accept this PR. --- docs/index.rst | 3 ++- psutil/_pslinux.py | 9 +++++++-- psutil/tests/test_linux.py | 26 ++++++++++++++++++++++++++ 3 files changed, 35 insertions(+), 3 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 4f77b2e49..0f55a2e1a 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -266,6 +266,7 @@ Memory - **active** *(UNIX)*: memory currently in use or very recently used, and so it is in RAM. - **inactive** *(UNIX)*: memory that is marked as not used. + - **slab** *(Linux)*: in-kernel data structures cache. - **buffers** *(Linux, BSD)*: cache for things like file system metadata. - **cached** *(Linux, BSD)*: cache for various things. - **shared** *(Linux, BSD)*: memory that may be simultaneously accessed by @@ -284,7 +285,7 @@ Memory >>> import psutil >>> mem = psutil.virtual_memory() >>> mem - svmem(total=10367352832, available=6472179712, percent=37.6, used=8186245120, free=2181107712, active=4748992512, inactive=2758115328, buffers=790724608, cached=3500347392, shared=787554304) + svmem(total=10367352832, available=6472179712, percent=37.6, used=8186245120, free=2181107712, active=4748992512, inactive=2758115328, buffers=790724608, cached=3500347392, shared=787554304, slab=199348224) >>> >>> THRESHOLD = 100 * 1024 * 1024 # 100MB >>> if mem.available <= THRESHOLD: diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index c8c10ec9b..0518ba04f 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -149,7 +149,7 @@ class IOPriority(enum.IntEnum): # psutil.virtual_memory() svmem = namedtuple( 'svmem', ['total', 'available', 'percent', 'used', 'free', - 'active', 'inactive', 'buffers', 'cached', 'shared']) + 'active', 'inactive', 'buffers', 'cached', 'shared', 'slab']) # psutil.disk_io_counters() sdiskio = namedtuple( 'sdiskio', ['read_count', 'write_count', @@ -441,6 +441,11 @@ def virtual_memory(): inactive = 0 missing_fields.append('inactive') + try: + slab = mems[b"Slab:"] + except KeyError: + slab = 0 + used = total - free - cached - buffers if used < 0: # May be symptomatic of running within a LCX container where such @@ -481,7 +486,7 @@ def virtual_memory(): warnings.warn(msg, RuntimeWarning) return svmem(total, avail, percent, used, free, - active, inactive, buffers, cached, shared) + active, inactive, buffers, cached, shared, slab) def swap_memory(): diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index 090634165..13303a540 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -237,6 +237,31 @@ def test_available(self): free_value, psutil_value, delta=MEMORY_TOLERANCE, msg='%s %s \n%s' % (free_value, psutil_value, out)) + def test_slab(self): + # Emulate /proc/meminfo because neither vmstat nor free return slab. + def open_mock(name, *args, **kwargs): + if name == '/proc/meminfo': + return io.BytesIO(textwrap.dedent("""\ + Active(anon): 6145416 kB + Active(file): 2950064 kB + Inactive(anon): 574764 kB + Inactive(file): 1567648 kB + MemAvailable: -1 kB + MemFree: 2057400 kB + MemTotal: 16325648 kB + SReclaimable: 346648 kB + Slab: 186836 kB + """).encode()) + else: + return orig_open(name, *args, **kwargs) + + orig_open = open + patch_point = 'builtins.open' if PY3 else '__builtin__.open' + with mock.patch(patch_point, create=True, side_effect=open_mock) as m: + ret = psutil.virtual_memory() + assert m.called + self.assertEqual(ret.slab, 191320064) + def test_warnings_on_misses(self): # Emulate a case where /proc/meminfo provides few info. # psutil is supposed to set the missing fields to 0 and @@ -280,6 +305,7 @@ def open_mock(name, *args, **kwargs): self.assertEqual(ret.shared, 0) self.assertEqual(ret.buffers, 0) self.assertEqual(ret.available, 0) + self.assertEqual(ret.slab, 0) def test_avail_old_percent(self): # Make sure that our calculation of avail mem for old kernels From 0cf3a5a22e65eacdca1f995af7497819de57c1d1 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 17 Mar 2018 10:47:05 +0100 Subject: [PATCH 034/182] update doc / HISTORY / CREDITS --- CREDITS | 4 ++++ HISTORY.rst | 5 +++++ docs/index.rst | 8 +++----- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/CREDITS b/CREDITS index f0327a7b3..ae384a530 100644 --- a/CREDITS +++ b/CREDITS @@ -527,3 +527,7 @@ I: 1243 N: Georg Sauthoff W: https://github.com/gsauthof I: 1193, 1194 + +N: Maxime Mouial +W: https://github.com/hush-hush +I: 1239 diff --git a/HISTORY.rst b/HISTORY.rst index 078ebff28..06cb3eee3 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -5,6 +5,11 @@ XXXX-XX-XX +**Enhancements** + +- 1239_: [Linux] expose kernel "slab" memory for psutil.virtual_memory(). + (patch by Maxime Mouial) + **Bug fixes** - 694_: [SunOS] cmdline() could be truncated at the 15th character when diff --git a/docs/index.rst b/docs/index.rst index 0f55a2e1a..395fd688f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -266,11 +266,11 @@ Memory - **active** *(UNIX)*: memory currently in use or very recently used, and so it is in RAM. - **inactive** *(UNIX)*: memory that is marked as not used. - - **slab** *(Linux)*: in-kernel data structures cache. - **buffers** *(Linux, BSD)*: cache for things like file system metadata. - **cached** *(Linux, BSD)*: cache for various things. - **shared** *(Linux, BSD)*: memory that may be simultaneously accessed by multiple processes. + - **slab** *(Linux)*: in-kernel data structures cache. - **wired** *(BSD, OSX)*: memory that is marked to always stay in RAM. It is never moved to disk. @@ -293,11 +293,9 @@ Memory ... >>> - .. versionchanged:: 4.2.0 added *shared* metrics on Linux. - - .. versionchanged:: 4.4.0 *available* and *used* values on Linux are more - precise and match "free" cmdline utility. + .. versionchanged:: 4.2.0 added *shared* metric on Linux. + .. versionchanged:: 5.4.4 added *slab* metric on Linux. .. function:: swap_memory() From 5d31cdd55f45b432fae01d6b0591682fdaedadf9 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 17 Mar 2018 11:03:45 +0100 Subject: [PATCH 035/182] suppress warnings from mock module --- psutil/tests/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 499240fb2..7be6fdc65 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -54,7 +54,9 @@ try: from unittest import mock # py3 except ImportError: - import mock # NOQA - requires "pip install mock" + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + import mock # NOQA - requires "pip install mock" if sys.version_info >= (3, 4): import enum From 091329c6522a6f53b6d79449693e0e4796ddab9c Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 17 Mar 2018 11:10:30 +0100 Subject: [PATCH 036/182] linux: add mock test for virtual_memory() --- psutil/tests/test_linux.py | 98 ++++++++++++++++++++++++++++---------- 1 file changed, 73 insertions(+), 25 deletions(-) diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index 13303a540..f4aefeeb4 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -237,31 +237,6 @@ def test_available(self): free_value, psutil_value, delta=MEMORY_TOLERANCE, msg='%s %s \n%s' % (free_value, psutil_value, out)) - def test_slab(self): - # Emulate /proc/meminfo because neither vmstat nor free return slab. - def open_mock(name, *args, **kwargs): - if name == '/proc/meminfo': - return io.BytesIO(textwrap.dedent("""\ - Active(anon): 6145416 kB - Active(file): 2950064 kB - Inactive(anon): 574764 kB - Inactive(file): 1567648 kB - MemAvailable: -1 kB - MemFree: 2057400 kB - MemTotal: 16325648 kB - SReclaimable: 346648 kB - Slab: 186836 kB - """).encode()) - else: - return orig_open(name, *args, **kwargs) - - orig_open = open - patch_point = 'builtins.open' if PY3 else '__builtin__.open' - with mock.patch(patch_point, create=True, side_effect=open_mock) as m: - ret = psutil.virtual_memory() - assert m.called - self.assertEqual(ret.slab, 191320064) - def test_warnings_on_misses(self): # Emulate a case where /proc/meminfo provides few info. # psutil is supposed to set the missing fields to 0 and @@ -425,6 +400,79 @@ def open_mock(name, *args, **kwargs): self.assertIn( "inactive memory stats couldn't be determined", str(w.message)) + def test_virtual_memory_mocked(self): + # Emulate /proc/meminfo because neither vmstat nor free return slab. + def open_mock(name, *args, **kwargs): + if name == '/proc/meminfo': + return io.BytesIO(textwrap.dedent("""\ + MemTotal: 100 kB + MemFree: 2 kB + MemAvailable: 3 kB + Buffers: 4 kB + Cached: 5 kB + SwapCached: 6 kB + Active: 7 kB + Inactive: 8 kB + Active(anon): 9 kB + Inactive(anon): 10 kB + Active(file): 11 kB + Inactive(file): 12 kB + Unevictable: 13 kB + Mlocked: 14 kB + SwapTotal: 15 kB + SwapFree: 16 kB + Dirty: 17 kB + Writeback: 18 kB + AnonPages: 19 kB + Mapped: 20 kB + Shmem: 21 kB + Slab: 22 kB + SReclaimable: 23 kB + SUnreclaim: 24 kB + KernelStack: 25 kB + PageTables: 26 kB + NFS_Unstable: 27 kB + Bounce: 28 kB + WritebackTmp: 29 kB + CommitLimit: 30 kB + Committed_AS: 31 kB + VmallocTotal: 32 kB + VmallocUsed: 33 kB + VmallocChunk: 34 kB + HardwareCorrupted: 35 kB + AnonHugePages: 36 kB + ShmemHugePages: 37 kB + ShmemPmdMapped: 38 kB + CmaTotal: 39 kB + CmaFree: 40 kB + HugePages_Total: 41 kB + HugePages_Free: 42 kB + HugePages_Rsvd: 43 kB + HugePages_Surp: 44 kB + Hugepagesize: 45 kB + DirectMap46k: 46 kB + DirectMap47M: 47 kB + DirectMap48G: 48 kB + """).encode()) + else: + return orig_open(name, *args, **kwargs) + + orig_open = open + patch_point = 'builtins.open' if PY3 else '__builtin__.open' + with mock.patch(patch_point, create=True, side_effect=open_mock) as m: + mem = psutil.virtual_memory() + assert m.called + self.assertEqual(mem.total, 100 * 1024) + self.assertEqual(mem.free, 2 * 1024) + self.assertEqual(mem.buffers, 4 * 1024) + # cached mem also includes reclaimable memory + self.assertEqual(mem.cached, (5 + 23) * 1024) + self.assertEqual(mem.shared, 21 * 1024) + self.assertEqual(mem.active, 7 * 1024) + self.assertEqual(mem.inactive, 8 * 1024) + self.assertEqual(mem.slab, 22 * 1024) + self.assertEqual(mem.available, 3 * 1024) + # ===================================================================== # --- system swap memory From 3b0de6bf54c7ec81d704e93b0d1cd44e1c92311a Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 17 Mar 2018 11:56:47 +0100 Subject: [PATCH 037/182] add a script to purge installation --- MANIFEST.in | 1 + Makefile | 3 ++- scripts/internal/purge.py | 42 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 1 deletion(-) create mode 100755 scripts/internal/purge.py diff --git a/MANIFEST.in b/MANIFEST.in index 7a92a4e5a..87d9bebb4 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -110,6 +110,7 @@ include scripts/internal/download_exes.py include scripts/internal/generate_manifest.py include scripts/internal/print_announce.py include scripts/internal/print_timeline.py +include scripts/internal/purge.py include scripts/internal/winmake.py include scripts/iotop.py include scripts/killall.py diff --git a/Makefile b/Makefile index 5081a4edd..d9f4a8541 100644 --- a/Makefile +++ b/Makefile @@ -76,7 +76,8 @@ install: ## Install this package as current user in "edit" mode. rm -rf tmp uninstall: ## Uninstall this package via pip. - cd ..; $(PYTHON) -m pip uninstall -y -v psutil + cd ..; $(PYTHON) -m pip uninstall -y -v psutil || true + $(PYTHON) scripts/internal/purge.py install-pip: ## Install pip (no-op if already installed). $(PYTHON) -c \ diff --git a/scripts/internal/purge.py b/scripts/internal/purge.py new file mode 100755 index 000000000..d93017195 --- /dev/null +++ b/scripts/internal/purge.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python + +# Copyright (c) 2009 Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +""" +Purge psutil installation by removing psutil-related files and +directories found in site-packages directories. This is needed mainly +because sometimes "import psutil" imports a leftover installation +from site-packages directory instead of the main working directory. +""" + +import os +import shutil +import site + + +PKGNAME = "psutil" + + +def rmpath(path): + if os.path.isdir(path): + print("rmdir " + path) + shutil.rmtree(path) + else: + print("rm " + path) + os.remove(path) + + +def main(): + locations = [site.getusersitepackages()] + locations.extend(site.getsitepackages()) + for root in locations: + if os.path.isdir(root): + for name in os.listdir(root): + if PKGNAME in name: + abspath = os.path.join(root, name) + rmpath(abspath) + + +main() From 329a5e2149848cf8c31f11b0525d6eede7def65b Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 17 Mar 2018 12:03:35 +0100 Subject: [PATCH 038/182] linux: disable unreliable test --- psutil/tests/test_linux.py | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index f4aefeeb4..be151d7c6 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -886,20 +886,21 @@ def ifconfig(nic): self.assertAlmostEqual( stats.dropout, ifconfig_ret['dropout'], delta=10) - @unittest.skipIf(not which('ip'), "'ip' utility not available") - @unittest.skipIf(TRAVIS, "skipped on Travis") - def test_net_if_names(self): - out = sh("ip addr").strip() - nics = [x for x in psutil.net_if_addrs().keys() if ':' not in x] - found = 0 - for line in out.split('\n'): - line = line.strip() - if re.search(r"^\d+:", line): - found += 1 - name = line.split(':')[1].strip() - self.assertIn(name, nics) - self.assertEqual(len(nics), found, msg="%s\n---\n%s" % ( - pprint.pformat(nics), out)) + # XXX - not reliable when having virtual NICs installed by Docker. + # @unittest.skipIf(not which('ip'), "'ip' utility not available") + # @unittest.skipIf(TRAVIS, "skipped on Travis") + # def test_net_if_names(self): + # out = sh("ip addr").strip() + # nics = [x for x in psutil.net_if_addrs().keys() if ':' not in x] + # found = 0 + # for line in out.split('\n'): + # line = line.strip() + # if re.search(r"^\d+:", line): + # found += 1 + # name = line.split(':')[1].strip() + # self.assertIn(name, nics) + # self.assertEqual(len(nics), found, msg="%s\n---\n%s" % ( + # pprint.pformat(nics), out)) @mock.patch('psutil._pslinux.socket.inet_ntop', side_effect=ValueError) @mock.patch('psutil._pslinux.supports_ipv6', return_value=False) From 81ef4c9185650213c03122c95856ad3991b5656d Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 17 Mar 2018 13:00:46 +0100 Subject: [PATCH 039/182] refactor open mock + add purge script (#1249) * update doc / HISTORY / CREDITS * suppress warnings from mock module * linux: add mock test for virtual_memory() * add a script to purge installation * linux: disable unreliable test * refactor open mock * refactor open mock * refactor open mock * refactor open mock * refactor open mock * refactor open mock * refactor open mock * refactor open mock * refactor open mock * refactor open mock * fix typo * fix typo * refactor open mock * refactor open mock * refactor open mock * refactor open mock * refactor open mock * add docstrings --- CREDITS | 4 + HISTORY.rst | 5 + MANIFEST.in | 1 + Makefile | 3 +- docs/index.rst | 8 +- psutil/tests/__init__.py | 4 +- psutil/tests/test_linux.py | 772 +++++++++++++++++-------------------- scripts/internal/purge.py | 42 ++ 8 files changed, 421 insertions(+), 418 deletions(-) create mode 100755 scripts/internal/purge.py diff --git a/CREDITS b/CREDITS index f0327a7b3..ae384a530 100644 --- a/CREDITS +++ b/CREDITS @@ -527,3 +527,7 @@ I: 1243 N: Georg Sauthoff W: https://github.com/gsauthof I: 1193, 1194 + +N: Maxime Mouial +W: https://github.com/hush-hush +I: 1239 diff --git a/HISTORY.rst b/HISTORY.rst index 078ebff28..06cb3eee3 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -5,6 +5,11 @@ XXXX-XX-XX +**Enhancements** + +- 1239_: [Linux] expose kernel "slab" memory for psutil.virtual_memory(). + (patch by Maxime Mouial) + **Bug fixes** - 694_: [SunOS] cmdline() could be truncated at the 15th character when diff --git a/MANIFEST.in b/MANIFEST.in index 7a92a4e5a..87d9bebb4 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -110,6 +110,7 @@ include scripts/internal/download_exes.py include scripts/internal/generate_manifest.py include scripts/internal/print_announce.py include scripts/internal/print_timeline.py +include scripts/internal/purge.py include scripts/internal/winmake.py include scripts/iotop.py include scripts/killall.py diff --git a/Makefile b/Makefile index 5081a4edd..d9f4a8541 100644 --- a/Makefile +++ b/Makefile @@ -76,7 +76,8 @@ install: ## Install this package as current user in "edit" mode. rm -rf tmp uninstall: ## Uninstall this package via pip. - cd ..; $(PYTHON) -m pip uninstall -y -v psutil + cd ..; $(PYTHON) -m pip uninstall -y -v psutil || true + $(PYTHON) scripts/internal/purge.py install-pip: ## Install pip (no-op if already installed). $(PYTHON) -c \ diff --git a/docs/index.rst b/docs/index.rst index 0f55a2e1a..395fd688f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -266,11 +266,11 @@ Memory - **active** *(UNIX)*: memory currently in use or very recently used, and so it is in RAM. - **inactive** *(UNIX)*: memory that is marked as not used. - - **slab** *(Linux)*: in-kernel data structures cache. - **buffers** *(Linux, BSD)*: cache for things like file system metadata. - **cached** *(Linux, BSD)*: cache for various things. - **shared** *(Linux, BSD)*: memory that may be simultaneously accessed by multiple processes. + - **slab** *(Linux)*: in-kernel data structures cache. - **wired** *(BSD, OSX)*: memory that is marked to always stay in RAM. It is never moved to disk. @@ -293,11 +293,9 @@ Memory ... >>> - .. versionchanged:: 4.2.0 added *shared* metrics on Linux. - - .. versionchanged:: 4.4.0 *available* and *used* values on Linux are more - precise and match "free" cmdline utility. + .. versionchanged:: 4.2.0 added *shared* metric on Linux. + .. versionchanged:: 5.4.4 added *slab* metric on Linux. .. function:: swap_memory() diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 499240fb2..7be6fdc65 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -54,7 +54,9 @@ try: from unittest import mock # py3 except ImportError: - import mock # NOQA - requires "pip install mock" + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + import mock # NOQA - requires "pip install mock" if sys.version_info >= (3, 4): import enum diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index 13303a540..dc9057963 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -13,7 +13,6 @@ import glob import io import os -import pprint import re import shutil import socket @@ -25,6 +24,7 @@ import psutil from psutil import LINUX +from psutil._compat import basestring from psutil._compat import PY3 from psutil._compat import u from psutil.tests import call_until @@ -143,6 +143,46 @@ def get_free_version_info(): return tuple(map(int, out.split()[-1].split('.'))) +@contextlib.contextmanager +def mock_open_content(for_path, content): + """Mock open() builtin and forces it to return a certain `content` + on read() if the path being opened matches `for_path`. + """ + def open_mock(name, *args, **kwargs): + if name == for_path: + if PY3: + if isinstance(content, basestring): + return io.StringIO(content) + else: + return io.BytesIO(content) + else: + return io.BytesIO(content) + else: + return orig_open(name, *args, **kwargs) + + orig_open = open + patch_point = 'builtins.open' if PY3 else '__builtin__.open' + with mock.patch(patch_point, create=True, side_effect=open_mock) as m: + yield m + + +@contextlib.contextmanager +def mock_open_exception(for_path, exc): + """Mock open() builtin and raises `exc` if the path being opened + matches `for_path`. + """ + def open_mock(name, *args, **kwargs): + if name == for_path: + raise exc + else: + return orig_open(name, *args, **kwargs) + + orig_open = open + patch_point = 'builtins.open' if PY3 else '__builtin__.open' + with mock.patch(patch_point, create=True, side_effect=open_mock) as m: + yield m + + # ===================================================================== # --- system virtual memory # ===================================================================== @@ -237,53 +277,22 @@ def test_available(self): free_value, psutil_value, delta=MEMORY_TOLERANCE, msg='%s %s \n%s' % (free_value, psutil_value, out)) - def test_slab(self): - # Emulate /proc/meminfo because neither vmstat nor free return slab. - def open_mock(name, *args, **kwargs): - if name == '/proc/meminfo': - return io.BytesIO(textwrap.dedent("""\ - Active(anon): 6145416 kB - Active(file): 2950064 kB - Inactive(anon): 574764 kB - Inactive(file): 1567648 kB - MemAvailable: -1 kB - MemFree: 2057400 kB - MemTotal: 16325648 kB - SReclaimable: 346648 kB - Slab: 186836 kB - """).encode()) - else: - return orig_open(name, *args, **kwargs) - - orig_open = open - patch_point = 'builtins.open' if PY3 else '__builtin__.open' - with mock.patch(patch_point, create=True, side_effect=open_mock) as m: - ret = psutil.virtual_memory() - assert m.called - self.assertEqual(ret.slab, 191320064) - def test_warnings_on_misses(self): # Emulate a case where /proc/meminfo provides few info. # psutil is supposed to set the missing fields to 0 and # raise a warning. - def open_mock(name, *args, **kwargs): - if name == '/proc/meminfo': - return io.BytesIO(textwrap.dedent("""\ - Active(anon): 6145416 kB - Active(file): 2950064 kB - Inactive(anon): 574764 kB - Inactive(file): 1567648 kB - MemAvailable: -1 kB - MemFree: 2057400 kB - MemTotal: 16325648 kB - SReclaimable: 346648 kB - """).encode()) - else: - return orig_open(name, *args, **kwargs) - - orig_open = open - patch_point = 'builtins.open' if PY3 else '__builtin__.open' - with mock.patch(patch_point, create=True, side_effect=open_mock) as m: + with mock_open_content( + '/proc/meminfo', + textwrap.dedent("""\ + Active(anon): 6145416 kB + Active(file): 2950064 kB + Inactive(anon): 574764 kB + Inactive(file): 1567648 kB + MemAvailable: -1 kB + MemFree: 2057400 kB + MemTotal: 16325648 kB + SReclaimable: 346648 kB + """).encode()) as m: with warnings.catch_warnings(record=True) as ws: warnings.simplefilter("always") ret = psutil.virtual_memory() @@ -328,29 +337,23 @@ def test_avail_old_percent(self): def test_avail_old_comes_from_kernel(self): # Make sure "MemAvailable:" coluimn is used instead of relying # on our internal algorithm to calculate avail mem. - def open_mock(name, *args, **kwargs): - if name == "/proc/meminfo": - return io.BytesIO(textwrap.dedent("""\ - Active: 9444728 kB - Active(anon): 6145416 kB - Active(file): 2950064 kB - Buffers: 287952 kB - Cached: 4818144 kB - Inactive(file): 1578132 kB - Inactive(anon): 574764 kB - Inactive(file): 1567648 kB - MemAvailable: 6574984 kB - MemFree: 2057400 kB - MemTotal: 16325648 kB - Shmem: 577588 kB - SReclaimable: 346648 kB - """).encode()) - else: - return orig_open(name, *args, **kwargs) - - orig_open = open - patch_point = 'builtins.open' if PY3 else '__builtin__.open' - with mock.patch(patch_point, create=True, side_effect=open_mock) as m: + with mock_open_content( + '/proc/meminfo', + textwrap.dedent("""\ + Active: 9444728 kB + Active(anon): 6145416 kB + Active(file): 2950064 kB + Buffers: 287952 kB + Cached: 4818144 kB + Inactive(file): 1578132 kB + Inactive(anon): 574764 kB + Inactive(file): 1567648 kB + MemAvailable: 6574984 kB + MemFree: 2057400 kB + MemTotal: 16325648 kB + Shmem: 577588 kB + SReclaimable: 346648 kB + """).encode()) as m: with warnings.catch_warnings(record=True) as ws: ret = psutil.virtual_memory() assert m.called @@ -363,9 +366,9 @@ def test_avail_old_missing_fields(self): # Remove Active(file), Inactive(file) and SReclaimable # from /proc/meminfo and make sure the fallback is used # (free + cached), - def open_mock(name, *args, **kwargs): - if name == "/proc/meminfo": - return io.BytesIO(textwrap.dedent("""\ + with mock_open_content( + "/proc/meminfo", + textwrap.dedent("""\ Active: 9444728 kB Active(anon): 6145416 kB Buffers: 287952 kB @@ -375,13 +378,7 @@ def open_mock(name, *args, **kwargs): MemFree: 2057400 kB MemTotal: 16325648 kB Shmem: 577588 kB - """).encode()) - else: - return orig_open(name, *args, **kwargs) - - orig_open = open - patch_point = 'builtins.open' if PY3 else '__builtin__.open' - with mock.patch(patch_point, create=True, side_effect=open_mock) as m: + """).encode()) as m: with warnings.catch_warnings(record=True) as ws: ret = psutil.virtual_memory() assert m.called @@ -393,9 +390,9 @@ def open_mock(name, *args, **kwargs): def test_avail_old_missing_zoneinfo(self): # Remove /proc/zoneinfo file. Make sure fallback is used # (free + cached). - def open_mock(name, *args, **kwargs): - if name == "/proc/meminfo": - return io.BytesIO(textwrap.dedent("""\ + with mock_open_content( + "/proc/meminfo", + textwrap.dedent("""\ Active: 9444728 kB Active(anon): 6145416 kB Active(file): 2950064 kB @@ -408,22 +405,85 @@ def open_mock(name, *args, **kwargs): MemTotal: 16325648 kB Shmem: 577588 kB SReclaimable: 346648 kB - """).encode()) - elif name == "/proc/zoneinfo": - raise IOError(errno.ENOENT, 'no such file or directory') - else: - return orig_open(name, *args, **kwargs) - - orig_open = open - patch_point = 'builtins.open' if PY3 else '__builtin__.open' - with mock.patch(patch_point, create=True, side_effect=open_mock) as m: - with warnings.catch_warnings(record=True) as ws: - ret = psutil.virtual_memory() + """).encode()): + with mock_open_exception( + "/proc/zoneinfo", + IOError(errno.ENOENT, 'no such file or directory')): + with warnings.catch_warnings(record=True) as ws: + ret = psutil.virtual_memory() + self.assertEqual( + ret.available, 2057400 * 1024 + 4818144 * 1024) + w = ws[0] + self.assertIn( + "inactive memory stats couldn't be determined", + str(w.message)) + + def test_virtual_memory_mocked(self): + # Emulate /proc/meminfo because neither vmstat nor free return slab. + with mock_open_content( + '/proc/meminfo', + textwrap.dedent("""\ + MemTotal: 100 kB + MemFree: 2 kB + MemAvailable: 3 kB + Buffers: 4 kB + Cached: 5 kB + SwapCached: 6 kB + Active: 7 kB + Inactive: 8 kB + Active(anon): 9 kB + Inactive(anon): 10 kB + Active(file): 11 kB + Inactive(file): 12 kB + Unevictable: 13 kB + Mlocked: 14 kB + SwapTotal: 15 kB + SwapFree: 16 kB + Dirty: 17 kB + Writeback: 18 kB + AnonPages: 19 kB + Mapped: 20 kB + Shmem: 21 kB + Slab: 22 kB + SReclaimable: 23 kB + SUnreclaim: 24 kB + KernelStack: 25 kB + PageTables: 26 kB + NFS_Unstable: 27 kB + Bounce: 28 kB + WritebackTmp: 29 kB + CommitLimit: 30 kB + Committed_AS: 31 kB + VmallocTotal: 32 kB + VmallocUsed: 33 kB + VmallocChunk: 34 kB + HardwareCorrupted: 35 kB + AnonHugePages: 36 kB + ShmemHugePages: 37 kB + ShmemPmdMapped: 38 kB + CmaTotal: 39 kB + CmaFree: 40 kB + HugePages_Total: 41 kB + HugePages_Free: 42 kB + HugePages_Rsvd: 43 kB + HugePages_Surp: 44 kB + Hugepagesize: 45 kB + DirectMap46k: 46 kB + DirectMap47M: 47 kB + DirectMap48G: 48 kB + """).encode()) as m: + mem = psutil.virtual_memory() assert m.called - self.assertEqual(ret.available, 2057400 * 1024 + 4818144 * 1024) - w = ws[0] - self.assertIn( - "inactive memory stats couldn't be determined", str(w.message)) + self.assertEqual(mem.total, 100 * 1024) + self.assertEqual(mem.free, 2 * 1024) + self.assertEqual(mem.buffers, 4 * 1024) + # cached mem also includes reclaimable memory + self.assertEqual(mem.cached, (5 + 23) * 1024) + self.assertEqual(mem.shared, 21 * 1024) + self.assertEqual(mem.active, 7 * 1024) + self.assertEqual(mem.inactive, 8 * 1024) + self.assertEqual(mem.slab, 22 * 1024) + self.assertEqual(mem.available, 3 * 1024) # ===================================================================== @@ -478,15 +538,9 @@ def test_missing_sin_sout(self): def test_no_vmstat_mocked(self): # see https://github.com/giampaolo/psutil/issues/722 - def open_mock(name, *args, **kwargs): - if name == "/proc/vmstat": - raise IOError(errno.ENOENT, 'no such file or directory') - else: - return orig_open(name, *args, **kwargs) - - orig_open = open - patch_point = 'builtins.open' if PY3 else '__builtin__.open' - with mock.patch(patch_point, create=True, side_effect=open_mock) as m: + with mock_open_exception( + "/proc/vmstat", + IOError(errno.ENOENT, 'no such file or directory')) as m: with warnings.catch_warnings(record=True) as ws: warnings.simplefilter("always") ret = psutil.swap_memory() @@ -521,15 +575,7 @@ def test_emulate_meminfo_has_no_metrics(self): # Emulate a case where /proc/meminfo provides no swap metrics # in which case sysinfo() syscall is supposed to be used # as a fallback. - def open_mock(name, *args, **kwargs): - if name == "/proc/meminfo": - return io.BytesIO(b"") - else: - return orig_open(name, *args, **kwargs) - - orig_open = open - patch_point = 'builtins.open' if PY3 else '__builtin__.open' - with mock.patch(patch_point, create=True, side_effect=open_mock) as m: + with mock_open_content("/proc/meminfo", b"") as m: psutil.swap_memory() assert m.called @@ -616,16 +662,9 @@ def test_cpu_count_logical_mocked(self): # Finally, let's make /proc/cpuinfo return meaningless data; # this way we'll fall back on relying on /proc/stat - def open_mock(name, *args, **kwargs): - if name.startswith('/proc/cpuinfo'): - return io.BytesIO(b"") - else: - return orig_open(name, *args, **kwargs) - - orig_open = open - patch_point = 'builtins.open' if PY3 else '__builtin__.open' - with mock.patch(patch_point, side_effect=open_mock, create=True): + with mock_open_content('/proc/cpuinfo', b"") as m: self.assertEqual(psutil._pslinux.cpu_count_logical(), original) + m.called def test_cpu_count_physical_mocked(self): # Have open() return emtpy data and make sure None is returned @@ -838,20 +877,21 @@ def ifconfig(nic): self.assertAlmostEqual( stats.dropout, ifconfig_ret['dropout'], delta=10) - @unittest.skipIf(not which('ip'), "'ip' utility not available") - @unittest.skipIf(TRAVIS, "skipped on Travis") - def test_net_if_names(self): - out = sh("ip addr").strip() - nics = [x for x in psutil.net_if_addrs().keys() if ':' not in x] - found = 0 - for line in out.split('\n'): - line = line.strip() - if re.search(r"^\d+:", line): - found += 1 - name = line.split(':')[1].strip() - self.assertIn(name, nics) - self.assertEqual(len(nics), found, msg="%s\n---\n%s" % ( - pprint.pformat(nics), out)) + # XXX - not reliable when having virtual NICs installed by Docker. + # @unittest.skipIf(not which('ip'), "'ip' utility not available") + # @unittest.skipIf(TRAVIS, "skipped on Travis") + # def test_net_if_names(self): + # out = sh("ip addr").strip() + # nics = [x for x in psutil.net_if_addrs().keys() if ':' not in x] + # found = 0 + # for line in out.split('\n'): + # line = line.strip() + # if re.search(r"^\d+:", line): + # found += 1 + # name = line.split(':')[1].strip() + # self.assertIn(name, nics) + # self.assertEqual(len(nics), found, msg="%s\n---\n%s" % ( + # pprint.pformat(nics), out)) @mock.patch('psutil._pslinux.socket.inet_ntop', side_effect=ValueError) @mock.patch('psutil._pslinux.supports_ipv6', return_value=False) @@ -866,20 +906,14 @@ def test_net_connections_ipv6_unsupported(self, supports_ipv6, inet_ntop): psutil.net_connections(kind='inet6') def test_net_connections_mocked(self): - def open_mock(name, *args, **kwargs): - if name == '/proc/net/unix': - return io.StringIO(textwrap.dedent(u"""\ - 0: 00000003 000 000 0001 03 462170 @/tmp/dbus-Qw2hMPIU3n - 0: 00000003 000 000 0001 03 35010 @/tmp/dbus-tB2X8h69BQ - 0: 00000003 000 000 0001 03 34424 @/tmp/dbus-cHy80Y8O - 000000000000000000000000000000000000000000000000000000 - """)) - else: - return orig_open(name, *args, **kwargs) - - orig_open = open - patch_point = 'builtins.open' if PY3 else '__builtin__.open' - with mock.patch(patch_point, side_effect=open_mock) as m: + with mock_open_content( + '/proc/net/unix', + textwrap.dedent("""\ + 0: 00000003 000 000 0001 03 462170 @/tmp/dbus-Qw2hMPIU3n + 0: 00000003 000 000 0001 03 35010 @/tmp/dbus-tB2X8h69BQ + 0: 00000003 000 000 0001 03 34424 @/tmp/dbus-cHy80Y8O + 000000000000000000000000000000000000000000000000000000 + """)) as m: psutil.net_connections(kind='unix') assert m.called @@ -945,65 +979,51 @@ def test_disk_partitions_mocked(self): def test_disk_io_counters_kernel_2_4_mocked(self): # Tests /proc/diskstats parsing format for 2.4 kernels, see: # https://github.com/giampaolo/psutil/issues/767 - def open_mock(name, *args, **kwargs): - if name == '/proc/partitions': - return io.StringIO(textwrap.dedent(u"""\ + with mock_open_content( + '/proc/partitions', + textwrap.dedent("""\ major minor #blocks name 8 0 488386584 hda - """)) - elif name == '/proc/diskstats': - return io.StringIO( - u(" 3 0 1 hda 2 3 4 5 6 7 8 9 10 11 12")) - else: - return orig_open(name, *args, **kwargs) - - orig_open = open - patch_point = 'builtins.open' if PY3 else '__builtin__.open' - with mock.patch(patch_point, side_effect=open_mock) as m: - ret = psutil.disk_io_counters(nowrap=False) - assert m.called - self.assertEqual(ret.read_count, 1) - self.assertEqual(ret.read_merged_count, 2) - self.assertEqual(ret.read_bytes, 3 * SECTOR_SIZE) - self.assertEqual(ret.read_time, 4) - self.assertEqual(ret.write_count, 5) - self.assertEqual(ret.write_merged_count, 6) - self.assertEqual(ret.write_bytes, 7 * SECTOR_SIZE) - self.assertEqual(ret.write_time, 8) - self.assertEqual(ret.busy_time, 10) + """)): + with mock_open_content( + '/proc/diskstats', + " 3 0 1 hda 2 3 4 5 6 7 8 9 10 11 12"): + ret = psutil.disk_io_counters(nowrap=False) + self.assertEqual(ret.read_count, 1) + self.assertEqual(ret.read_merged_count, 2) + self.assertEqual(ret.read_bytes, 3 * SECTOR_SIZE) + self.assertEqual(ret.read_time, 4) + self.assertEqual(ret.write_count, 5) + self.assertEqual(ret.write_merged_count, 6) + self.assertEqual(ret.write_bytes, 7 * SECTOR_SIZE) + self.assertEqual(ret.write_time, 8) + self.assertEqual(ret.busy_time, 10) def test_disk_io_counters_kernel_2_6_full_mocked(self): # Tests /proc/diskstats parsing format for 2.6 kernels, # lines reporting all metrics: # https://github.com/giampaolo/psutil/issues/767 - def open_mock(name, *args, **kwargs): - if name == '/proc/partitions': - return io.StringIO(textwrap.dedent(u"""\ + with mock_open_content( + '/proc/partitions', + textwrap.dedent("""\ major minor #blocks name 8 0 488386584 hda - """)) - elif name == '/proc/diskstats': - return io.StringIO( - u(" 3 0 hda 1 2 3 4 5 6 7 8 9 10 11")) - else: - return orig_open(name, *args, **kwargs) - - orig_open = open - patch_point = 'builtins.open' if PY3 else '__builtin__.open' - with mock.patch(patch_point, side_effect=open_mock) as m: - ret = psutil.disk_io_counters(nowrap=False) - assert m.called - self.assertEqual(ret.read_count, 1) - self.assertEqual(ret.read_merged_count, 2) - self.assertEqual(ret.read_bytes, 3 * SECTOR_SIZE) - self.assertEqual(ret.read_time, 4) - self.assertEqual(ret.write_count, 5) - self.assertEqual(ret.write_merged_count, 6) - self.assertEqual(ret.write_bytes, 7 * SECTOR_SIZE) - self.assertEqual(ret.write_time, 8) - self.assertEqual(ret.busy_time, 10) + """)): + with mock_open_content( + '/proc/diskstats', + " 3 0 hda 1 2 3 4 5 6 7 8 9 10 11"): + ret = psutil.disk_io_counters(nowrap=False) + self.assertEqual(ret.read_count, 1) + self.assertEqual(ret.read_merged_count, 2) + self.assertEqual(ret.read_bytes, 3 * SECTOR_SIZE) + self.assertEqual(ret.read_time, 4) + self.assertEqual(ret.write_count, 5) + self.assertEqual(ret.write_merged_count, 6) + self.assertEqual(ret.write_bytes, 7 * SECTOR_SIZE) + self.assertEqual(ret.write_time, 8) + self.assertEqual(ret.busy_time, 10) def test_disk_io_counters_kernel_2_6_limited_mocked(self): # Tests /proc/diskstats parsing format for 2.6 kernels, @@ -1011,34 +1031,27 @@ def test_disk_io_counters_kernel_2_6_limited_mocked(self): # amount of metrics when it bumps into a partition # (instead of a disk). See: # https://github.com/giampaolo/psutil/issues/767 - def open_mock(name, *args, **kwargs): - if name == '/proc/partitions': - return io.StringIO(textwrap.dedent(u"""\ + with mock_open_content( + '/proc/partitions', + textwrap.dedent("""\ major minor #blocks name 8 0 488386584 hda - """)) - elif name == '/proc/diskstats': - return io.StringIO( - u(" 3 1 hda 1 2 3 4")) - else: - return orig_open(name, *args, **kwargs) - - orig_open = open - patch_point = 'builtins.open' if PY3 else '__builtin__.open' - with mock.patch(patch_point, side_effect=open_mock) as m: - ret = psutil.disk_io_counters(nowrap=False) - assert m.called - self.assertEqual(ret.read_count, 1) - self.assertEqual(ret.read_bytes, 2 * SECTOR_SIZE) - self.assertEqual(ret.write_count, 3) - self.assertEqual(ret.write_bytes, 4 * SECTOR_SIZE) - - self.assertEqual(ret.read_merged_count, 0) - self.assertEqual(ret.read_time, 0) - self.assertEqual(ret.write_merged_count, 0) - self.assertEqual(ret.write_time, 0) - self.assertEqual(ret.busy_time, 0) + """)): + with mock_open_content( + '/proc/diskstats', + " 3 1 hda 1 2 3 4"): + ret = psutil.disk_io_counters(nowrap=False) + self.assertEqual(ret.read_count, 1) + self.assertEqual(ret.read_bytes, 2 * SECTOR_SIZE) + self.assertEqual(ret.write_count, 3) + self.assertEqual(ret.write_bytes, 4 * SECTOR_SIZE) + + self.assertEqual(ret.read_merged_count, 0) + self.assertEqual(ret.read_time, 0) + self.assertEqual(ret.write_merged_count, 0) + self.assertEqual(ret.write_time, 0) + self.assertEqual(ret.busy_time, 0) # ===================================================================== @@ -1119,20 +1132,13 @@ def open_mock(name, *args, **kwargs): def test_cpu_steal_decrease(self): # Test cumulative cpu stats decrease. We should ignore this. # See issue #1210. - - def open_mock(name, *args, **kwargs): - if name == "/proc/stat": - return io.BytesIO(textwrap.dedent("""\ - cpu 0 0 0 0 0 0 0 1 0 0 - cpu0 0 0 0 0 0 0 0 1 0 0 - cpu1 0 0 0 0 0 0 0 1 0 0 - """).encode()) - return orig_open(name, *args, **kwargs) - - orig_open = open - patch_point = 'builtins.open' if PY3 else '__builtin__.open' - - with mock.patch(patch_point, create=True, side_effect=open_mock) as m: + with mock_open_content( + "/proc/stat", + textwrap.dedent("""\ + cpu 0 0 0 0 0 0 0 1 0 0 + cpu0 0 0 0 0 0 0 0 1 0 0 + cpu1 0 0 0 0 0 0 0 1 0 0 + """).encode()) as m: # first call to "percent" functions should read the new stat file # and compare to the "real" file read at import time - so the # values are meaningless @@ -1142,16 +1148,13 @@ def open_mock(name, *args, **kwargs): psutil.cpu_times_percent() psutil.cpu_times_percent(percpu=True) - def open_mock(name, *args, **kwargs): - if name == "/proc/stat": - return io.BytesIO(textwrap.dedent("""\ - cpu 1 0 0 0 0 0 0 0 0 0 - cpu0 1 0 0 0 0 0 0 0 0 0 - cpu1 1 0 0 0 0 0 0 0 0 0 - """).encode()) - return orig_open(name, *args, **kwargs) - - with mock.patch(patch_point, create=True, side_effect=open_mock) as m: + with mock_open_content( + "/proc/stat", + textwrap.dedent("""\ + cpu 1 0 0 0 0 0 0 0 0 0 + cpu0 1 0 0 0 0 0 0 0 0 0 + cpu1 1 0 0 0 0 0 0 0 0 0 + """).encode()) as m: # Increase "user" while steal goes "backwards" to zero. cpu_percent = psutil.cpu_percent() assert m.called @@ -1254,16 +1257,9 @@ def test_pid_exists_no_proc_status(self): # Internally pid_exists relies on /proc/{pid}/status. # Emulate a case where this file is empty in which case # psutil is supposed to fall back on using pids(). - def open_mock(name, *args, **kwargs): - if name == "/proc/%s/status" % os.getpid(): - return io.StringIO(u("")) - else: - return orig_open(name, *args, **kwargs) - - orig_open = open - patch_point = 'builtins.open' if PY3 else '__builtin__.open' - with mock.patch(patch_point, side_effect=open_mock): + with mock_open_content("/proc/%s/status", "") as m: assert psutil.pid_exists(os.getpid()) + assert m.called # ===================================================================== @@ -1378,51 +1374,33 @@ def open_mock(name, *args, **kwargs): def test_emulate_no_base_files(self): # Emulate a case where base metrics files are not present, # in which case we're supposed to get None. - def open_mock(name, *args, **kwargs): - if name.startswith("/sys/class/power_supply/BAT0/energy_now") or \ - name.startswith("/sys/class/power_supply/BAT0/charge_now"): - raise IOError(errno.ENOENT, "") - else: - return orig_open(name, *args, **kwargs) - - orig_open = open - patch_point = 'builtins.open' if PY3 else '__builtin__.open' - with mock.patch(patch_point, side_effect=open_mock) as m: - self.assertIsNone(psutil.sensors_battery()) - assert m.called + with mock_open_exception( + "/sys/class/power_supply/BAT0/energy_now", + IOError(errno.ENOENT, "")): + with mock_open_exception( + "/sys/class/power_supply/BAT0/charge_now", + IOError(errno.ENOENT, "")): + self.assertIsNone(psutil.sensors_battery()) def test_emulate_energy_full_0(self): # Emulate a case where energy_full files returns 0. - def open_mock(name, *args, **kwargs): - if name.startswith("/sys/class/power_supply/BAT0/energy_full"): - return io.BytesIO(b"0") - else: - return orig_open(name, *args, **kwargs) - - orig_open = open - patch_point = 'builtins.open' if PY3 else '__builtin__.open' - with mock.patch(patch_point, side_effect=open_mock) as m: + with mock_open_content( + "/sys/class/power_supply/BAT0/energy_full", b"0") as m: self.assertEqual(psutil.sensors_battery().percent, 0) assert m.called def test_emulate_energy_full_not_avail(self): # Emulate a case where energy_full file does not exist. # Expected fallback on /capacity. - def open_mock(name, *args, **kwargs): - energy_full = "/sys/class/power_supply/BAT0/energy_full" - charge_full = "/sys/class/power_supply/BAT0/charge_full" - if name.startswith(energy_full) or name.startswith(charge_full): - raise IOError(errno.ENOENT, "") - elif name.startswith("/sys/class/power_supply/BAT0/capacity"): - return io.BytesIO(b"88") - else: - return orig_open(name, *args, **kwargs) - - orig_open = open - patch_point = 'builtins.open' if PY3 else '__builtin__.open' - with mock.patch(patch_point, side_effect=open_mock) as m: - self.assertEqual(psutil.sensors_battery().percent, 88) - assert m.called + with mock_open_exception( + "/sys/class/power_supply/BAT0/energy_full", + IOError(errno.ENOENT, "")): + with mock_open_exception( + "/sys/class/power_supply/BAT0/charge_full", + IOError(errno.ENOENT, "")): + with mock_open_content( + "/sys/class/power_supply/BAT0/capacity", b"88"): + self.assertEqual(psutil.sensors_battery().percent, 88) def test_emulate_no_ac0_online(self): # Emulate a case where /AC0/online file does not exist. @@ -1440,19 +1418,16 @@ def path_exists_mock(name): def test_emulate_no_power(self): # Emulate a case where /AC0/online file nor /BAT0/status exist. - def open_mock(name, *args, **kwargs): - if name.startswith("/sys/class/power_supply/AC/online") or \ - name.startswith("/sys/class/power_supply/AC0/online") or \ - name.startswith("/sys/class/power_supply/BAT0/status"): - raise IOError(errno.ENOENT, "") - else: - return orig_open(name, *args, **kwargs) - - orig_open = open - patch_point = 'builtins.open' if PY3 else '__builtin__.open' - with mock.patch(patch_point, side_effect=open_mock) as m: - self.assertIsNone(psutil.sensors_battery().power_plugged) - assert m.called + with mock_open_exception( + "/sys/class/power_supply/AC/online", + IOError(errno.ENOENT, "")): + with mock_open_exception( + "/sys/class/power_supply/AC0/online", + IOError(errno.ENOENT, "")): + with mock_open_exception( + "/sys/class/power_supply/BAT0/status", + IOError(errno.ENOENT, "")): + self.assertIsNone(psutil.sensors_battery().power_plugged) @unittest.skipIf(not LINUX, "LINUX only") @@ -1561,37 +1536,31 @@ def test_memory_full_info(self): def test_memory_full_info_mocked(self): # See: https://github.com/giampaolo/psutil/issues/1222 - def open_mock(name, *args, **kwargs): - if name == "/proc/%s/smaps" % os.getpid(): - return io.BytesIO(textwrap.dedent("""\ - fffff0 r-xp 00000000 00:00 0 [vsyscall] - Size: 1 kB - Rss: 2 kB - Pss: 3 kB - Shared_Clean: 4 kB - Shared_Dirty: 5 kB - Private_Clean: 6 kB - Private_Dirty: 7 kB - Referenced: 8 kB - Anonymous: 9 kB - LazyFree: 10 kB - AnonHugePages: 11 kB - ShmemPmdMapped: 12 kB - Shared_Hugetlb: 13 kB - Private_Hugetlb: 14 kB - Swap: 15 kB - SwapPss: 16 kB - KernelPageSize: 17 kB - MMUPageSize: 18 kB - Locked: 19 kB - VmFlags: rd ex - """).encode()) - else: - return orig_open(name, *args, **kwargs) - - orig_open = open - patch_point = 'builtins.open' if PY3 else '__builtin__.open' - with mock.patch(patch_point, create=True, side_effect=open_mock) as m: + with mock_open_content( + "/proc/%s/smaps" % os.getpid(), + textwrap.dedent("""\ + fffff0 r-xp 00000000 00:00 0 [vsyscall] + Size: 1 kB + Rss: 2 kB + Pss: 3 kB + Shared_Clean: 4 kB + Shared_Dirty: 5 kB + Private_Clean: 6 kB + Private_Dirty: 7 kB + Referenced: 8 kB + Anonymous: 9 kB + LazyFree: 10 kB + AnonHugePages: 11 kB + ShmemPmdMapped: 12 kB + Shared_Hugetlb: 13 kB + Private_Hugetlb: 14 kB + Swap: 15 kB + SwapPss: 16 kB + KernelPageSize: 17 kB + MMUPageSize: 18 kB + Locked: 19 kB + VmFlags: rd ex + """).encode()) as m: p = psutil.Process() mem = p.memory_full_info() assert m.called @@ -1775,15 +1744,9 @@ def test_exe_mocked(self): def test_issue_1014(self): # Emulates a case where smaps file does not exist. In this case # wrap_exception decorator should not raise NoSuchProcess. - def open_mock(name, *args, **kwargs): - if name.startswith('/proc/%s/smaps' % os.getpid()): - raise IOError(errno.ENOENT, "") - else: - return orig_open(name, *args, **kwargs) - - orig_open = open - patch_point = 'builtins.open' if PY3 else '__builtin__.open' - with mock.patch(patch_point, side_effect=open_mock) as m: + with mock_open_exception( + '/proc/%s/smaps' % os.getpid(), + IOError(errno.ENOENT, "")) as m: p = psutil.Process() with self.assertRaises(IOError) as err: p.memory_maps() @@ -1819,56 +1782,49 @@ def test_cwd_zombie(self): def test_stat_file_parsing(self): from psutil._pslinux import CLOCK_TICKS - def open_mock(name, *args, **kwargs): - if name.startswith('/proc/%s/stat' % os.getpid()): - args = [ - "0", # pid - "(cat)", # name - "Z", # status - "1", # ppid - "0", # pgrp - "0", # session - "0", # tty - "0", # tpgid - "0", # flags - "0", # minflt - "0", # cminflt - "0", # majflt - "0", # cmajflt - "2", # utime - "3", # stime - "4", # cutime - "5", # cstime - "0", # priority - "0", # nice - "0", # num_threads - "0", # itrealvalue - "6", # starttime - "0", # vsize - "0", # rss - "0", # rsslim - "0", # startcode - "0", # endcode - "0", # startstack - "0", # kstkesp - "0", # kstkeip - "0", # signal - "0", # blocked - "0", # sigignore - "0", # sigcatch - "0", # wchan - "0", # nswap - "0", # cnswap - "0", # exit_signal - "6", # processor - ] - return io.BytesIO(" ".join(args).encode()) - else: - return orig_open(name, *args, **kwargs) - - orig_open = open - patch_point = 'builtins.open' if PY3 else '__builtin__.open' - with mock.patch(patch_point, side_effect=open_mock): + args = [ + "0", # pid + "(cat)", # name + "Z", # status + "1", # ppid + "0", # pgrp + "0", # session + "0", # tty + "0", # tpgid + "0", # flags + "0", # minflt + "0", # cminflt + "0", # majflt + "0", # cmajflt + "2", # utime + "3", # stime + "4", # cutime + "5", # cstime + "0", # priority + "0", # nice + "0", # num_threads + "0", # itrealvalue + "6", # starttime + "0", # vsize + "0", # rss + "0", # rsslim + "0", # startcode + "0", # endcode + "0", # startstack + "0", # kstkesp + "0", # kstkeip + "0", # signal + "0", # blocked + "0", # sigignore + "0", # sigcatch + "0", # wchan + "0", # nswap + "0", # cnswap + "0", # exit_signal + "6", # processor + ] + content = " ".join(args).encode() + with mock_open_content('/proc/%s/stat' % os.getpid(), content): p = psutil.Process() self.assertEqual(p.name(), 'cat') self.assertEqual(p.status(), psutil.STATUS_ZOMBIE) @@ -1883,22 +1839,16 @@ def open_mock(name, *args, **kwargs): self.assertEqual(p.cpu_num(), 6) def test_status_file_parsing(self): - def open_mock(name, *args, **kwargs): - if name.startswith('/proc/%s/status' % os.getpid()): - return io.BytesIO(textwrap.dedent("""\ - Uid:\t1000\t1001\t1002\t1003 - Gid:\t1004\t1005\t1006\t1007 - Threads:\t66 - Cpus_allowed:\tf - Cpus_allowed_list:\t0-7 - voluntary_ctxt_switches:\t12 - nonvoluntary_ctxt_switches:\t13""").encode()) - else: - return orig_open(name, *args, **kwargs) - - orig_open = open - patch_point = 'builtins.open' if PY3 else '__builtin__.open' - with mock.patch(patch_point, side_effect=open_mock): + with mock_open_content( + '/proc/%s/status' % os.getpid(), + textwrap.dedent("""\ + Uid:\t1000\t1001\t1002\t1003 + Gid:\t1004\t1005\t1006\t1007 + Threads:\t66 + Cpus_allowed:\tf + Cpus_allowed_list:\t0-7 + voluntary_ctxt_switches:\t12 + nonvoluntary_ctxt_switches:\t13""").encode()): p = psutil.Process() self.assertEqual(p.num_ctx_switches().voluntary, 12) self.assertEqual(p.num_ctx_switches().involuntary, 13) diff --git a/scripts/internal/purge.py b/scripts/internal/purge.py new file mode 100755 index 000000000..d93017195 --- /dev/null +++ b/scripts/internal/purge.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python + +# Copyright (c) 2009 Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +""" +Purge psutil installation by removing psutil-related files and +directories found in site-packages directories. This is needed mainly +because sometimes "import psutil" imports a leftover installation +from site-packages directory instead of the main working directory. +""" + +import os +import shutil +import site + + +PKGNAME = "psutil" + + +def rmpath(path): + if os.path.isdir(path): + print("rmdir " + path) + shutil.rmtree(path) + else: + print("rm " + path) + os.remove(path) + + +def main(): + locations = [site.getusersitepackages()] + locations.extend(site.getsitepackages()) + for root in locations: + if os.path.isdir(root): + for name in os.listdir(root): + if PKGNAME in name: + abspath = os.path.join(root, name) + rmpath(abspath) + + +main() From 95b8e4ba7c2f90d6239266890212feb4031c6bed Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 20 Mar 2018 19:37:37 +0100 Subject: [PATCH 040/182] fix #1238: sensors_battery() may return None in case battery is not listed as BAT0 under /sys/class/power_supply. --- HISTORY.rst | 2 ++ psutil/_pslinux.py | 8 ++++++-- psutil/tests/__init__.py | 2 +- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 06cb3eee3..5279f2031 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -29,6 +29,8 @@ XXXX-XX-XX - 1216_: fix compatibility with python 2.6 on Windows (patch by Dan Vinakovsky) - 1222_: [Linux] Process.memory_full_info() was erroneously summing "Swap:" and "SwapPss:". Same for "Pss:" and "SwapPss". Not anymore. +- 1238_: [Linux] sensors_battery() may return None in case battery is not + listed as "BAT0" under /sys/class/power_supply. - 1240_: [Windows] cpu_times() float loses accuracy in a long running system. (patch by stswandering) - 1245_: [Linux] sensors_temperatures() may fail with IOError "no such file". diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index 0518ba04f..9f45410f5 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -1225,9 +1225,13 @@ def multi_cat(*paths): return int(ret) if ret.isdigit() else ret return None - root = os.path.join(POWER_SUPPLY_PATH, "BAT0") - if not os.path.exists(root): + bats = [x for x in os.listdir(POWER_SUPPLY_PATH) if x.startswith('BAT')] + if not bats: return None + # Get the first available battery. Usually this is "BAT0", except + # some rare exceptions: + # https://github.com/giampaolo/psutil/issues/1238 + root = os.path.join(POWER_SUPPLY_PATH, sorted(bats)[0]) # Base metrics. energy_now = multi_cat( diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 7be6fdc65..537f78170 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -162,7 +162,7 @@ HAS_RLIMIT = hasattr(psutil.Process, "rlimit") HAS_THREADS = hasattr(psutil.Process, "threads") HAS_SENSORS_BATTERY = hasattr(psutil, "sensors_battery") -HAS_BATTERY = HAS_SENSORS_BATTERY and psutil.sensors_battery() +HAS_BATTERY = HAS_SENSORS_BATTERY and bool(psutil.sensors_battery()) HAS_SENSORS_FANS = hasattr(psutil, "sensors_fans") HAS_SENSORS_TEMPERATURES = hasattr(psutil, "sensors_temperatures") From cfc67c0b153816cb53d81eb944b8cb4ed0be2d20 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 24 Mar 2018 12:24:59 +0100 Subject: [PATCH 041/182] fix git commit hook cmd on win --- Makefile | 2 +- scripts/internal/winmake.py | 12 +++++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index d9f4a8541..f7b4cd7f5 100644 --- a/Makefile +++ b/Makefile @@ -264,7 +264,7 @@ bench-oneshot-2: ## Same as above but using perf module (supposed to be more pr $(TEST_PREFIX) $(PYTHON) scripts/internal/bench_oneshot_2.py check-broken-links: ## Look for broken links in source files. - git ls-files | xargs $(PYTHON) -Wa scripts/internal/check_broken_links.py + git ls-files | xargs $(PYTHON) -Wa scripts/internal/check_broken_links.py help: ## Display callable targets. @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' diff --git a/scripts/internal/winmake.py b/scripts/internal/winmake.py index 548f7a8ed..19c27ed1a 100755 --- a/scripts/internal/winmake.py +++ b/scripts/internal/winmake.py @@ -28,6 +28,8 @@ TSCRIPT = os.getenv('TSCRIPT', 'psutil\\tests\\__main__.py') GET_PIP_URL = "https://bootstrap.pypa.io/get-pip.py" PY3 = sys.version_info[0] == 3 +HERE = os.path.abspath(os.path.dirname(__file__)) +ROOT_DIR = os.path.realpath(os.path.join(HERE, "..", "..")) DEPS = [ "coverage", "flake8", @@ -442,18 +444,26 @@ def test_memleaks(): @cmd def install_git_hooks(): + """Install GIT pre-commit hook.""" if os.path.isdir('.git'): - shutil.copy(".git-pre-commit", ".git\\hooks\\pre-commit") + src = os.path.join(ROOT_DIR, ".git-pre-commit") + dst = os.path.realpath( + os.path.join(ROOT_DIR, ".git", "hooks", "pre-commit")) + with open(src, "rt") as s: + with open(dst, "wt") as d: + d.write(s.read()) @cmd def bench_oneshot(): + """Benchmarks for oneshot() ctx manager (see #799).""" install() sh("%s -Wa scripts\\internal\\bench_oneshot.py" % PYTHON) @cmd def bench_oneshot_2(): + """Same as above but using perf module (supposed to be more precise).""" install() sh("%s -Wa scripts\\internal\\bench_oneshot_2.py" % PYTHON) From 3ef2f9a792c9616ba9eacc0fc7ce8e66a30fc8a7 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 24 Mar 2018 15:35:18 -0400 Subject: [PATCH 042/182] 1224 fix windows Process.wait() (#1253) * define C TimeoutExpired exception * fix git commit hook cmd on win * raise TimeoutAbandoned on WAIT_ABANDONED and rely on a pure-python internal polling * update HISTORY * update HISTORY * update comments * update comments --- HISTORY.rst | 1 + Makefile | 2 +- psutil/_psutil_windows.c | 36 ++++++++++++++++++++++++---- psutil/_pswindows.py | 47 ++++++++++++++++++++++++++++--------- scripts/internal/winmake.py | 12 +++++++++- 5 files changed, 81 insertions(+), 17 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 5279f2031..d0bb4b0a8 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -29,6 +29,7 @@ XXXX-XX-XX - 1216_: fix compatibility with python 2.6 on Windows (patch by Dan Vinakovsky) - 1222_: [Linux] Process.memory_full_info() was erroneously summing "Swap:" and "SwapPss:". Same for "Pss:" and "SwapPss". Not anymore. +- 1224_: [Windows] Process.wait() may erroneously raise TimeoutExpired. - 1238_: [Linux] sensors_battery() may return None in case battery is not listed as "BAT0" under /sys/class/power_supply. - 1240_: [Windows] cpu_times() float loses accuracy in a long running system. diff --git a/Makefile b/Makefile index d9f4a8541..f7b4cd7f5 100644 --- a/Makefile +++ b/Makefile @@ -264,7 +264,7 @@ bench-oneshot-2: ## Same as above but using perf module (supposed to be more pr $(TEST_PREFIX) $(PYTHON) scripts/internal/bench_oneshot_2.py check-broken-links: ## Look for broken links in source files. - git ls-files | xargs $(PYTHON) -Wa scripts/internal/check_broken_links.py + git ls-files | xargs $(PYTHON) -Wa scripts/internal/check_broken_links.py help: ## Display callable targets. @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' diff --git a/psutil/_psutil_windows.c b/psutil/_psutil_windows.c index f164168af..bb993dbd4 100644 --- a/psutil/_psutil_windows.c +++ b/psutil/_psutil_windows.c @@ -201,6 +201,9 @@ psutil_get_nic_addresses() { * ============================================================================ */ +// Raised by Process.wait(). +static PyObject *TimeoutExpired; +static PyObject *TimeoutAbandoned; static ULONGLONG (*psutil_GetTickCount64)(void) = NULL; @@ -400,20 +403,33 @@ psutil_proc_wait(PyObject *self, PyObject *args) { retVal = WaitForSingleObject(hProcess, timeout); Py_END_ALLOW_THREADS + // handle return code if (retVal == WAIT_FAILED) { CloseHandle(hProcess); - return PyErr_SetFromWindowsErr(0); + PyErr_SetFromWindowsErr(0); + return NULL; } if (retVal == WAIT_TIMEOUT) { CloseHandle(hProcess); - return Py_BuildValue("l", WAIT_TIMEOUT); + PyErr_SetString(TimeoutExpired, + "WaitForSingleObject() returned WAIT_TIMEOUT"); + return NULL; + } + if (retVal == WAIT_ABANDONED) { + psutil_debug("WaitForSingleObject() -> WAIT_ABANDONED"); + CloseHandle(hProcess); + PyErr_SetString(TimeoutAbandoned, + "WaitForSingleObject() returned WAIT_ABANDONED"); + return NULL; } + // WaitForSingleObject() returned WAIT_OBJECT_0. It means the + // process is gone so we can get its process exit code. The PID + // may still stick around though but we'll handle that from Python. if (GetExitCodeProcess(hProcess, &ExitCode) == 0) { CloseHandle(hProcess); return PyErr_SetFromWindowsErr(GetLastError()); } - CloseHandle(hProcess); #if PY_MAJOR_VERSION >= 3 @@ -2847,7 +2863,7 @@ psutil_proc_info(PyObject *self, PyObject *args) { (double)process->UserTime.LowPart * LO_T; kernel_time = (double)process->KernelTime.HighPart * HI_T + \ (double)process->KernelTime.LowPart * LO_T; - + // Convert the LARGE_INTEGER union to a Unix time. // It's the best I could find by googling and borrowing code here // and there. The time returned has a precision of 1 second. @@ -3768,6 +3784,18 @@ void init_psutil_windows(void) INITERROR; } + // Exceptions. + TimeoutExpired = PyErr_NewException( + "_psutil_windows.TimeoutExpired", NULL, NULL); + Py_INCREF(TimeoutExpired); + PyModule_AddObject(module, "TimeoutExpired", TimeoutExpired); + + TimeoutAbandoned = PyErr_NewException( + "_psutil_windows.TimeoutAbandoned", NULL, NULL); + Py_INCREF(TimeoutAbandoned); + PyModule_AddObject(module, "TimeoutAbandoned", TimeoutAbandoned); + + // version constant PyModule_AddIntConstant(module, "version", PSUTIL_VERSION); // process status constants diff --git a/psutil/_pswindows.py b/psutil/_pswindows.py index 0eb4b14f8..bb95d2a0d 100644 --- a/psutil/_pswindows.py +++ b/psutil/_pswindows.py @@ -9,6 +9,7 @@ import functools import os import sys +import time from collections import namedtuple from . import _common @@ -78,7 +79,6 @@ # ===================================================================== CONN_DELETE_TCB = "DELETE_TCB" -WAIT_TIMEOUT = 0x00000102 # 258 in decimal ACCESS_DENIED_ERRSET = frozenset([errno.EPERM, errno.EACCES, cext.ERROR_ACCESS_DENIED]) NO_SUCH_SERVICE_ERRSET = frozenset([cext.ERROR_INVALID_NAME, @@ -792,18 +792,43 @@ def wait(self, timeout=None): if timeout is None: cext_timeout = cext.INFINITE else: - # WaitForSingleObject() expects time in milliseconds + # WaitForSingleObject() expects time in milliseconds. cext_timeout = int(timeout * 1000) + + timer = getattr(time, 'monotonic', time.time) + stop_at = timer() + timeout if timeout is not None else None + + try: + # Exit code is supposed to come from GetExitCodeProcess(). + # May also be None if OpenProcess() failed with + # ERROR_INVALID_PARAMETER, meaning PID is already gone. + exit_code = cext.proc_wait(self.pid, cext_timeout) + except cext.TimeoutExpired: + # WaitForSingleObject() returned WAIT_TIMEOUT. Just raise. + raise TimeoutExpired(timeout, self.pid, self._name) + except cext.TimeoutAbandoned: + # WaitForSingleObject() returned WAIT_ABANDONED, see: + # https://github.com/giampaolo/psutil/issues/1224 + # We'll just rely on the internal polling and return None + # when the PID disappears. Subprocess module does the same + # (return None): + # https://github.com/python/cpython/blob/ + # be50a7b627d0aa37e08fa8e2d5568891f19903ce/ + # Lib/subprocess.py#L1193-L1194 + exit_code = None + + # At this point WaitForSingleObject() returned WAIT_OBJECT_0, + # meaning the process is gone. Stupidly there are cases where + # its PID may still stick around so we do a further internal + # polling. + delay = 0.0001 while True: - ret = cext.proc_wait(self.pid, cext_timeout) - if ret == WAIT_TIMEOUT: - raise TimeoutExpired(timeout, self.pid, self._name) - if pid_exists(self.pid): - if timeout is None: - continue - else: - raise TimeoutExpired(timeout, self.pid, self._name) - return ret + if not pid_exists(self.pid): + return exit_code + if stop_at and timer() >= stop_at: + raise TimeoutExpired(timeout, pid=self.pid, name=self._name) + time.sleep(delay) + delay = min(delay * 2, 0.04) # incremental delay @wrap_exceptions def username(self): diff --git a/scripts/internal/winmake.py b/scripts/internal/winmake.py index 548f7a8ed..19c27ed1a 100755 --- a/scripts/internal/winmake.py +++ b/scripts/internal/winmake.py @@ -28,6 +28,8 @@ TSCRIPT = os.getenv('TSCRIPT', 'psutil\\tests\\__main__.py') GET_PIP_URL = "https://bootstrap.pypa.io/get-pip.py" PY3 = sys.version_info[0] == 3 +HERE = os.path.abspath(os.path.dirname(__file__)) +ROOT_DIR = os.path.realpath(os.path.join(HERE, "..", "..")) DEPS = [ "coverage", "flake8", @@ -442,18 +444,26 @@ def test_memleaks(): @cmd def install_git_hooks(): + """Install GIT pre-commit hook.""" if os.path.isdir('.git'): - shutil.copy(".git-pre-commit", ".git\\hooks\\pre-commit") + src = os.path.join(ROOT_DIR, ".git-pre-commit") + dst = os.path.realpath( + os.path.join(ROOT_DIR, ".git", "hooks", "pre-commit")) + with open(src, "rt") as s: + with open(dst, "wt") as d: + d.write(s.read()) @cmd def bench_oneshot(): + """Benchmarks for oneshot() ctx manager (see #799).""" install() sh("%s -Wa scripts\\internal\\bench_oneshot.py" % PYTHON) @cmd def bench_oneshot_2(): + """Same as above but using perf module (supposed to be more precise).""" install() sh("%s -Wa scripts\\internal\\bench_oneshot_2.py" % PYTHON) From bf79914769f0e9e41830adccc09c62636a66f0dd Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 24 Mar 2018 20:48:44 +0100 Subject: [PATCH 043/182] rename function arg --- psutil/_common.py | 6 +++--- psutil/_psaix.py | 4 ++-- psutil/_psbsd.py | 4 ++-- psutil/_pslinux.py | 4 ++-- psutil/_psosx.py | 4 ++-- psutil/_psposix.py | 2 +- psutil/_pssunos.py | 4 ++-- psutil/_pswindows.py | 6 +++--- 8 files changed, 17 insertions(+), 17 deletions(-) diff --git a/psutil/_common.py b/psutil/_common.py index 870971e41..05dbb4ce6 100644 --- a/psutil/_common.py +++ b/psutil/_common.py @@ -261,14 +261,14 @@ class BatteryTime(enum.IntEnum): # =================================================================== -def usage_percent(used, total, _round=None): +def usage_percent(used, total, round_=None): """Calculate percentage usage of 'used' against 'total'.""" try: ret = (used / total) * 100 except ZeroDivisionError: ret = 0.0 if isinstance(used, float) or isinstance(total, float) else 0 - if _round is not None: - return round(ret, _round) + if round_ is not None: + return round(ret, round_) else: return ret diff --git a/psutil/_psaix.py b/psutil/_psaix.py index 9abc8d17e..662f306c3 100644 --- a/psutil/_psaix.py +++ b/psutil/_psaix.py @@ -117,7 +117,7 @@ def get_procfs_path(): def virtual_memory(): total, avail, free, pinned, inuse = cext.virtual_mem() - percent = usage_percent((total - avail), total, _round=1) + percent = usage_percent((total - avail), total, round_=1) return svmem(total, avail, percent, inuse, free) @@ -125,7 +125,7 @@ def swap_memory(): """Swap system memory as a (total, used, free, sin, sout) tuple.""" total, free, sin, sout = cext.swap_mem() used = total - free - percent = usage_percent(used, total, _round=1) + percent = usage_percent(used, total, round_=1) return _common.sswap(total, used, free, percent, sin, sout) diff --git a/psutil/_psbsd.py b/psutil/_psbsd.py index 0553401a5..83f38d55e 100644 --- a/psutil/_psbsd.py +++ b/psutil/_psbsd.py @@ -188,7 +188,7 @@ def virtual_memory(): shared = int(line.split()[1]) * 1024 avail = inactive + cached + free used = active + wired + cached - percent = usage_percent((total - avail), total, _round=1) + percent = usage_percent((total - avail), total, round_=1) return svmem(total, avail, percent, used, free, active, inactive, buffers, cached, shared, wired) @@ -196,7 +196,7 @@ def virtual_memory(): def swap_memory(): """System swap memory as (total, used, free, sin, sout) namedtuple.""" total, used, free, sin, sout = cext.swap_mem() - percent = usage_percent(used, total, _round=1) + percent = usage_percent(used, total, round_=1) return _common.sswap(total, used, free, percent, sin, sout) diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index 9f45410f5..78c03d5c5 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -476,7 +476,7 @@ def virtual_memory(): if avail > total: avail = free - percent = usage_percent((total - avail), total, _round=1) + percent = usage_percent((total - avail), total, round_=1) # Warn about missing metrics which are set to 0. if missing_fields: @@ -509,7 +509,7 @@ def swap_memory(): free *= unit_multiplier used = total - free - percent = usage_percent(used, total, _round=1) + percent = usage_percent(used, total, round_=1) # get pgin/pgouts try: f = open_binary("%s/vmstat" % get_procfs_path()) diff --git a/psutil/_psosx.py b/psutil/_psosx.py index 308756a81..193f63e00 100644 --- a/psutil/_psosx.py +++ b/psutil/_psosx.py @@ -122,7 +122,7 @@ def virtual_memory(): total, active, inactive, wired, free = cext.virtual_mem() avail = inactive + free used = active + inactive + wired - percent = usage_percent((total - avail), total, _round=1) + percent = usage_percent((total - avail), total, round_=1) return svmem(total, avail, percent, used, free, active, inactive, wired) @@ -130,7 +130,7 @@ def virtual_memory(): def swap_memory(): """Swap system memory as a (total, used, free, sin, sout) tuple.""" total, used, free, sin, sout = cext.swap_mem() - percent = usage_percent(used, total, _round=1) + percent = usage_percent(used, total, round_=1) return _common.sswap(total, used, free, percent, sin, sout) diff --git a/psutil/_psposix.py b/psutil/_psposix.py index 6bb8444d8..9c3fac27e 100644 --- a/psutil/_psposix.py +++ b/psutil/_psposix.py @@ -156,7 +156,7 @@ def disk_usage(path): # User usage percent compared to the total amount of space # the user can use. This number would be higher if compared # to root's because the user has less space (usually -5%). - usage_percent_user = usage_percent(used, total_user, _round=1) + usage_percent_user = usage_percent(used, total_user, round_=1) # NB: the percentage is -5% than what shown by df due to # reserved blocks that we are currently not considering: diff --git a/psutil/_pssunos.py b/psutil/_pssunos.py index 35b4b092b..e2f33a3ae 100644 --- a/psutil/_pssunos.py +++ b/psutil/_pssunos.py @@ -131,7 +131,7 @@ def virtual_memory(): # note: there's no difference on Solaris free = avail = os.sysconf('SC_AVPHYS_PAGES') * PAGE_SIZE used = total - free - percent = usage_percent(used, total, _round=1) + percent = usage_percent(used, total, round_=1) return svmem(total, avail, percent, used, free) @@ -163,7 +163,7 @@ def swap_memory(): total += int(int(t) * 512) free += int(int(f) * 512) used = total - free - percent = usage_percent(used, total, _round=1) + percent = usage_percent(used, total, round_=1) return _common.sswap(total, used, free, percent, sin * PAGE_SIZE, sout * PAGE_SIZE) diff --git a/psutil/_pswindows.py b/psutil/_pswindows.py index bb95d2a0d..ab727cba3 100644 --- a/psutil/_pswindows.py +++ b/psutil/_pswindows.py @@ -217,7 +217,7 @@ def virtual_memory(): avail = availphys free = availphys used = total - avail - percent = usage_percent((total - avail), total, _round=1) + percent = usage_percent((total - avail), total, round_=1) return svmem(total, avail, percent, used, free) @@ -227,7 +227,7 @@ def swap_memory(): total = mem[2] free = mem[3] used = total - free - percent = usage_percent(used, total, _round=1) + percent = usage_percent(used, total, round_=1) return _common.sswap(total, used, free, percent, 0, 0) @@ -247,7 +247,7 @@ def disk_usage(path): path = path.decode(ENCODING, errors="strict") total, free = cext.disk_usage(path) used = total - free - percent = usage_percent(used, total, _round=1) + percent = usage_percent(used, total, round_=1) return _common.sdiskusage(total, used, free, percent) From 504aff59a47f785e67d736f141a6fc8262651cc9 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 24 Mar 2018 20:49:44 +0100 Subject: [PATCH 044/182] remove duplicate test --- psutil/tests/test_linux.py | 67 -------------------------------------- 1 file changed, 67 deletions(-) diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index 50ce795e6..d4eaf2a14 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -418,73 +418,6 @@ def test_avail_old_missing_zoneinfo(self): "inactive memory stats couldn't be determined", str(w.message)) - def test_virtual_memory_mocked(self): - # Emulate /proc/meminfo because neither vmstat nor free return slab. - with mock_open_content( - '/proc/meminfo', - textwrap.dedent("""\ - MemTotal: 100 kB - MemFree: 2 kB - MemAvailable: 3 kB - Buffers: 4 kB - Cached: 5 kB - SwapCached: 6 kB - Active: 7 kB - Inactive: 8 kB - Active(anon): 9 kB - Inactive(anon): 10 kB - Active(file): 11 kB - Inactive(file): 12 kB - Unevictable: 13 kB - Mlocked: 14 kB - SwapTotal: 15 kB - SwapFree: 16 kB - Dirty: 17 kB - Writeback: 18 kB - AnonPages: 19 kB - Mapped: 20 kB - Shmem: 21 kB - Slab: 22 kB - SReclaimable: 23 kB - SUnreclaim: 24 kB - KernelStack: 25 kB - PageTables: 26 kB - NFS_Unstable: 27 kB - Bounce: 28 kB - WritebackTmp: 29 kB - CommitLimit: 30 kB - Committed_AS: 31 kB - VmallocTotal: 32 kB - VmallocUsed: 33 kB - VmallocChunk: 34 kB - HardwareCorrupted: 35 kB - AnonHugePages: 36 kB - ShmemHugePages: 37 kB - ShmemPmdMapped: 38 kB - CmaTotal: 39 kB - CmaFree: 40 kB - HugePages_Total: 41 kB - HugePages_Free: 42 kB - HugePages_Rsvd: 43 kB - HugePages_Surp: 44 kB - Hugepagesize: 45 kB - DirectMap46k: 46 kB - DirectMap47M: 47 kB - DirectMap48G: 48 kB - """).encode()) as m: - mem = psutil.virtual_memory() - assert m.called - self.assertEqual(mem.total, 100 * 1024) - self.assertEqual(mem.free, 2 * 1024) - self.assertEqual(mem.buffers, 4 * 1024) - # cached mem also includes reclaimable memory - self.assertEqual(mem.cached, (5 + 23) * 1024) - self.assertEqual(mem.shared, 21 * 1024) - self.assertEqual(mem.active, 7 * 1024) - self.assertEqual(mem.inactive, 8 * 1024) - self.assertEqual(mem.slab, 22 * 1024) - self.assertEqual(mem.available, 3 * 1024) - def test_virtual_memory_mocked(self): # Emulate /proc/meminfo because neither vmstat nor free return slab. def open_mock(name, *args, **kwargs): From 043da3edd9484c30c84162d819b0bc60766738db Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 28 Mar 2018 15:47:35 +0200 Subject: [PATCH 045/182] appveyor: do not use only_commit files directive as it no longer works --- appveyor.yml | 44 +++++++++++++++++++++++--------------------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 28f5f7f6f..4cb693990 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -100,24 +100,26 @@ cache: skip_commits: message: skip-ci -# run build only if one of the following files is modified on commit -only_commits: - files: - .ci/appveyor/* - appveyor.yml - psutil/__init__.py - psutil/_common.py - psutil/_compat.py - psutil/_psutil_common.* - psutil/_psutil_windows.* - psutil/_pswindows.py - psutil/arch/windows/* - psutil/tests/__init__.py - psutil/tests/__main__.py - psutil/tests/test_memory_leaks.py - psutil/tests/test_misc.py - psutil/tests/test_process.py - psutil/tests/test_system.py - psutil/tests/test_windows.py - scripts/* - setup.py +# Commented as per: +# https://help.appveyor.com/discussions/problems/13269 +# # run build only if one of the following files is modified on commit +# only_commits: +# files: +# .ci/appveyor/* +# appveyor.yml +# psutil/__init__.py +# psutil/_common.py +# psutil/_compat.py +# psutil/_psutil_common.* +# psutil/_psutil_windows.* +# psutil/_pswindows.py +# psutil/arch/windows/* +# psutil/tests/__init__.py +# psutil/tests/__main__.py +# psutil/tests/test_memory_leaks.py +# psutil/tests/test_misc.py +# psutil/tests/test_process.py +# psutil/tests/test_system.py +# psutil/tests/test_windows.py +# scripts/* +# setup.py From 3544efc5db6131ea7dc9cd5c90c5db31f519a86d Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 28 Mar 2018 17:09:17 +0200 Subject: [PATCH 046/182] fix appveyor failure, see https://ci.appveyor.com/project/giampaolo/psutil/build/job/6in5bk62ekiploys --- appveyor.yml | 2 +- psutil/tests/__init__.py | 4 +++- psutil/tests/test_process.py | 4 ++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 4cb693990..e1f6c3551 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -70,7 +70,7 @@ install: - "powershell .ci\\appveyor\\install.ps1" # - ps: (new-object net.webclient).DownloadFile('https://raw.github.com/pypa/pip/master/contrib/get-pip.py', 'C:/get-pip.py') - "%WITH_COMPILER% %PYTHON%/python.exe -m pip --version" - - "%WITH_COMPILER% %PYTHON%/python.exe -m pip install --upgrade --user unittest2 ipaddress pypiwin32 wmi wheel" + - "%WITH_COMPILER% %PYTHON%/python.exe -m pip install --upgrade --user unittest2 ipaddress pypiwin32==220 wmi wheel" - "%WITH_COMPILER% %PYTHON%/python.exe -m pip freeze" - "%WITH_COMPILER% %PYTHON%/python.exe scripts/internal/winmake.py clean" - "%WITH_COMPILER% %PYTHON%/python.exe setup.py build" diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 537f78170..dcdbd4fa8 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -482,7 +482,9 @@ def assert_gone(pid): try: subp.terminate() except OSError as err: - if err.errno != errno.ESRCH: + if WINDOWS and err.errno == 6: # "invalid handle" + pass + elif err.errno != errno.ESRCH: raise if subp.stdout: subp.stdout.close() diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index 33557b051..3411114af 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -1206,9 +1206,9 @@ def test_halfway_terminated_process(self): # Test that NoSuchProcess exception gets raised in case the # process dies after we create the Process object. # Example: - # >>> proc = Process(1234) + # >>> proc = Process(1234) # >>> time.sleep(2) # time-consuming task, process dies in meantime - # >>> proc.name() + # >>> proc.name() # Refers to Issue #15 sproc = get_test_subprocess() p = psutil.Process(sproc.pid) From bc1156f5280690ddd9c8e9aa5dcba22bc668f72c Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 28 Mar 2018 20:16:49 +0200 Subject: [PATCH 047/182] try to fix appveyor --- appveyor.yml | 48 ++++++++++++++++++++++++------------------------ 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index e1f6c3551..e5197a62b 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -70,7 +70,7 @@ install: - "powershell .ci\\appveyor\\install.ps1" # - ps: (new-object net.webclient).DownloadFile('https://raw.github.com/pypa/pip/master/contrib/get-pip.py', 'C:/get-pip.py') - "%WITH_COMPILER% %PYTHON%/python.exe -m pip --version" - - "%WITH_COMPILER% %PYTHON%/python.exe -m pip install --upgrade --user unittest2 ipaddress pypiwin32==220 wmi wheel" + - "%WITH_COMPILER% %PYTHON%/python.exe -m pip install --upgrade --user unittest2 ipaddress pypiwin32==219 wmi wheel" - "%WITH_COMPILER% %PYTHON%/python.exe -m pip freeze" - "%WITH_COMPILER% %PYTHON%/python.exe scripts/internal/winmake.py clean" - "%WITH_COMPILER% %PYTHON%/python.exe setup.py build" @@ -100,26 +100,26 @@ cache: skip_commits: message: skip-ci -# Commented as per: -# https://help.appveyor.com/discussions/problems/13269 -# # run build only if one of the following files is modified on commit -# only_commits: -# files: -# .ci/appveyor/* -# appveyor.yml -# psutil/__init__.py -# psutil/_common.py -# psutil/_compat.py -# psutil/_psutil_common.* -# psutil/_psutil_windows.* -# psutil/_pswindows.py -# psutil/arch/windows/* -# psutil/tests/__init__.py -# psutil/tests/__main__.py -# psutil/tests/test_memory_leaks.py -# psutil/tests/test_misc.py -# psutil/tests/test_process.py -# psutil/tests/test_system.py -# psutil/tests/test_windows.py -# scripts/* -# setup.py +Commented as per: +https://help.appveyor.com/discussions/problems/13269 +# run build only if one of the following files is modified on commit +only_commits: + files: + .ci/appveyor/* + appveyor.yml + psutil/__init__.py + psutil/_common.py + psutil/_compat.py + psutil/_psutil_common.* + psutil/_psutil_windows.* + psutil/_pswindows.py + psutil/arch/windows/* + psutil/tests/__init__.py + psutil/tests/__main__.py + psutil/tests/test_memory_leaks.py + psutil/tests/test_misc.py + psutil/tests/test_process.py + psutil/tests/test_system.py + psutil/tests/test_windows.py + scripts/* + setup.py From 9ae36307d18d3c673513f07ac29e9b490f0a1009 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 28 Mar 2018 20:17:19 +0200 Subject: [PATCH 048/182] try to fix appveyor --- appveyor.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index e5197a62b..1791e83e0 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -100,8 +100,6 @@ cache: skip_commits: message: skip-ci -Commented as per: -https://help.appveyor.com/discussions/problems/13269 # run build only if one of the following files is modified on commit only_commits: files: From e7fc5348e415397b86e299c5ddf3b85d7312cafb Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 29 Mar 2018 17:06:42 +0200 Subject: [PATCH 049/182] add oshi project --- IDEAS | 1 + 1 file changed, 1 insertion(+) diff --git a/IDEAS b/IDEAS index 4932ad728..903726983 100644 --- a/IDEAS +++ b/IDEAS @@ -163,3 +163,4 @@ RESOURCES - zabbix: https://zabbix.org/wiki/Get_Zabbix - libstatgrab: http://www.i-scream.org/libstatgrab/ - top: http://www.unixtop.org/ +- oshi: https://github.com/oshi/oshi From 79a256a8162d10090f273944945d290d6e11396f Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 29 Mar 2018 17:10:12 +0200 Subject: [PATCH 050/182] fix appveyor.yml syntax --- appveyor.yml | 36 ++++++++++++++++++------------------ scripts/internal/winmake.py | 2 +- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 1791e83e0..de505f6a5 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -103,21 +103,21 @@ skip_commits: # run build only if one of the following files is modified on commit only_commits: files: - .ci/appveyor/* - appveyor.yml - psutil/__init__.py - psutil/_common.py - psutil/_compat.py - psutil/_psutil_common.* - psutil/_psutil_windows.* - psutil/_pswindows.py - psutil/arch/windows/* - psutil/tests/__init__.py - psutil/tests/__main__.py - psutil/tests/test_memory_leaks.py - psutil/tests/test_misc.py - psutil/tests/test_process.py - psutil/tests/test_system.py - psutil/tests/test_windows.py - scripts/* - setup.py + - .ci/appveyor/* + - appveyor.yml + - psutil/__init__.py + - psutil/_common.py + - psutil/_compat.py + - psutil/_psutil_common.* + - psutil/_psutil_windows.* + - psutil/_pswindows.py + - psutil/arch/windows/* + - psutil/tests/__init__.py + - psutil/tests/__main__.py + - psutil/tests/test_memory_leaks.py + - psutil/tests/test_misc.py + - psutil/tests/test_process.py + - psutil/tests/test_system.py + - psutil/tests/test_windows.py + - scripts/* + - setup.py diff --git a/scripts/internal/winmake.py b/scripts/internal/winmake.py index 19c27ed1a..575437508 100755 --- a/scripts/internal/winmake.py +++ b/scripts/internal/winmake.py @@ -213,7 +213,7 @@ def build(): @cmd -def build_wheel(): +def wheel(): """Create wheel file.""" build() sh("%s setup.py bdist_wheel" % PYTHON) From 07edb8746ac651ed8346791802e598ee4c29dd75 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 29 Mar 2018 17:16:22 +0200 Subject: [PATCH 051/182] try to fix appveyor / setuptools --- appveyor.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/appveyor.yml b/appveyor.yml index de505f6a5..1bacc0651 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -70,6 +70,7 @@ install: - "powershell .ci\\appveyor\\install.ps1" # - ps: (new-object net.webclient).DownloadFile('https://raw.github.com/pypa/pip/master/contrib/get-pip.py', 'C:/get-pip.py') - "%WITH_COMPILER% %PYTHON%/python.exe -m pip --version" + - "%WITH_COMPILER% %PYTHON%/python.exe -m pip install --upgrade --user setuptools pip" - "%WITH_COMPILER% %PYTHON%/python.exe -m pip install --upgrade --user unittest2 ipaddress pypiwin32==219 wmi wheel" - "%WITH_COMPILER% %PYTHON%/python.exe -m pip freeze" - "%WITH_COMPILER% %PYTHON%/python.exe scripts/internal/winmake.py clean" From 1feb2d48e39b83e01372d592e15bac2f4f1b0c3d Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 29 Mar 2018 17:50:57 +0200 Subject: [PATCH 052/182] try to fix appveyor failure --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 1bacc0651..6c35a5d51 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -71,7 +71,7 @@ install: # - ps: (new-object net.webclient).DownloadFile('https://raw.github.com/pypa/pip/master/contrib/get-pip.py', 'C:/get-pip.py') - "%WITH_COMPILER% %PYTHON%/python.exe -m pip --version" - "%WITH_COMPILER% %PYTHON%/python.exe -m pip install --upgrade --user setuptools pip" - - "%WITH_COMPILER% %PYTHON%/python.exe -m pip install --upgrade --user unittest2 ipaddress pypiwin32==219 wmi wheel" + - "%WITH_COMPILER% %PYTHON%/python.exe -m pip install --upgrade --user unittest2 ipaddress pypiwin32==220 wmi wheel" - "%WITH_COMPILER% %PYTHON%/python.exe -m pip freeze" - "%WITH_COMPILER% %PYTHON%/python.exe scripts/internal/winmake.py clean" - "%WITH_COMPILER% %PYTHON%/python.exe setup.py build" From bf9f8ff572d946b069660278cfe15054b05804e1 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 29 Mar 2018 19:20:10 +0200 Subject: [PATCH 053/182] appveyor: use a pywin32 version which works with python 3.4 --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 6c35a5d51..1bacc0651 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -71,7 +71,7 @@ install: # - ps: (new-object net.webclient).DownloadFile('https://raw.github.com/pypa/pip/master/contrib/get-pip.py', 'C:/get-pip.py') - "%WITH_COMPILER% %PYTHON%/python.exe -m pip --version" - "%WITH_COMPILER% %PYTHON%/python.exe -m pip install --upgrade --user setuptools pip" - - "%WITH_COMPILER% %PYTHON%/python.exe -m pip install --upgrade --user unittest2 ipaddress pypiwin32==220 wmi wheel" + - "%WITH_COMPILER% %PYTHON%/python.exe -m pip install --upgrade --user unittest2 ipaddress pypiwin32==219 wmi wheel" - "%WITH_COMPILER% %PYTHON%/python.exe -m pip freeze" - "%WITH_COMPILER% %PYTHON%/python.exe scripts/internal/winmake.py clean" - "%WITH_COMPILER% %PYTHON%/python.exe setup.py build" From a4c07e9da0f7a3a92b27c856eec09954ccc824ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Denis=20Krienb=C3=BChl?= Date: Thu, 29 Mar 2018 20:12:10 +0200 Subject: [PATCH 054/182] Fixes swap_memory not returning bytes on FreeBSD (#1260) --- psutil/arch/freebsd/specific.c | 10 ++++++---- psutil/tests/test_bsd.py | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/psutil/arch/freebsd/specific.c b/psutil/arch/freebsd/specific.c index 2c8944ddd..f6d1a67ea 100644 --- a/psutil/arch/freebsd/specific.c +++ b/psutil/arch/freebsd/specific.c @@ -516,10 +516,12 @@ psutil_swap_mem(PyObject *self, PyObject *args) { if (sysctlbyname("vm.stats.vm.v_vnodeout", &nodeout, &size, NULL, 0) == -1) goto error; - return Py_BuildValue("(iiiII)", - kvmsw[0].ksw_total, // total - kvmsw[0].ksw_used, // used - kvmsw[0].ksw_total - kvmsw[0].ksw_used, // free + int pagesize = getpagesize(); + + return Py_BuildValue("(KKKII)", + (unsigned long long) kvmsw[0].ksw_total * pagesize, // total + (unsigned long long) kvmsw[0].ksw_used * pagesize, // used + (unsigned long long) kvmsw[0].ksw_total * pagesize - kvmsw[0].ksw_used * pagesize, // free swapin + swapout, // swap in nodein + nodeout); // swap out diff --git a/psutil/tests/test_bsd.py b/psutil/tests/test_bsd.py index d3868ada1..63cae66af 100755 --- a/psutil/tests/test_bsd.py +++ b/psutil/tests/test_bsd.py @@ -344,6 +344,40 @@ def test_cpu_stats_syscalls(self): # self.assertAlmostEqual(psutil.cpu_stats().traps, # sysctl('vm.stats.sys.v_trap'), delta=1000) + # --- swap memory + @staticmethod + def parse_swapinfo(): + + # the last line is always the total + output = sh("swapinfo -k").splitlines()[-1] + parts = re.split(r'\s+', output) + + if not parts: + raise ValueError("Can't parse swapinfo: %s" % output) + + # the size is in 1k units, so multiply by 1024 + total, used, free = (int(p) * 1024 for p in parts[1:4]) + + return total, used, free + + def test_swapmem_free(self): + total, used, free = self.parse_swapinfo() + + self.assertAlmostEqual( + psutil.swap_memory().free, free, delta=MEMORY_TOLERANCE) + + def test_swapmem_used(self): + total, used, free = self.parse_swapinfo() + + self.assertAlmostEqual( + psutil.swap_memory().used, used, delta=MEMORY_TOLERANCE) + + def test_swapmem_total(self): + total, used, free = self.parse_swapinfo() + + self.assertAlmostEqual( + psutil.swap_memory().total, total, delta=MEMORY_TOLERANCE) + # --- others def test_boot_time(self): From 904252e99a9d3217eda01159ab83e9384db54590 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 29 Mar 2018 20:27:01 +0200 Subject: [PATCH 055/182] #1255 adjust style + give CREDITS to @href --- CREDITS | 4 ++++ HISTORY.rst | 2 ++ psutil/arch/freebsd/specific.c | 15 +++++++++------ psutil/tests/test_bsd.py | 30 +++++++++++++----------------- 4 files changed, 28 insertions(+), 23 deletions(-) diff --git a/CREDITS b/CREDITS index ae384a530..41061cdf7 100644 --- a/CREDITS +++ b/CREDITS @@ -531,3 +531,7 @@ I: 1193, 1194 N: Maxime Mouial W: https://github.com/hush-hush I: 1239 + +N: Denis Krienbühl +W: https://github.com/href +I: 1260 diff --git a/HISTORY.rst b/HISTORY.rst index d0bb4b0a8..65cad58e0 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -35,6 +35,8 @@ XXXX-XX-XX - 1240_: [Windows] cpu_times() float loses accuracy in a long running system. (patch by stswandering) - 1245_: [Linux] sensors_temperatures() may fail with IOError "no such file". +- 1255_: [FreeBSD] swap_memory() stats were erroneously represented in KB. + (patch by Denis Krienbühl) 5.4.3 ===== diff --git a/psutil/arch/freebsd/specific.c b/psutil/arch/freebsd/specific.c index f6d1a67ea..cf0b7df24 100644 --- a/psutil/arch/freebsd/specific.c +++ b/psutil/arch/freebsd/specific.c @@ -518,12 +518,15 @@ psutil_swap_mem(PyObject *self, PyObject *args) { int pagesize = getpagesize(); - return Py_BuildValue("(KKKII)", - (unsigned long long) kvmsw[0].ksw_total * pagesize, // total - (unsigned long long) kvmsw[0].ksw_used * pagesize, // used - (unsigned long long) kvmsw[0].ksw_total * pagesize - kvmsw[0].ksw_used * pagesize, // free - swapin + swapout, // swap in - nodein + nodeout); // swap out + return Py_BuildValue( + "(KKKII)", + (unsigned long long)kvmsw[0].ksw_total * pagesize, // total + (unsigned long long)kvmsw[0].ksw_used * pagesize, // used + (unsigned long long)kvmsw[0].ksw_total * pagesize - // free + kvmsw[0].ksw_used * pagesize, + swapin + swapout, // swap in + nodein + nodeout // swap out + ); error: return PyErr_SetFromErrno(PyExc_OSError); diff --git a/psutil/tests/test_bsd.py b/psutil/tests/test_bsd.py index 63cae66af..7846c1ca2 100755 --- a/psutil/tests/test_bsd.py +++ b/psutil/tests/test_bsd.py @@ -159,6 +159,19 @@ def setUpClass(cls): def tearDownClass(cls): reap_children() + @staticmethod + def parse_swapinfo(): + # the last line is always the total + output = sh("swapinfo -k").splitlines()[-1] + parts = re.split(r'\s+', output) + + if not parts: + raise ValueError("Can't parse swapinfo: %s" % output) + + # the size is in 1k units, so multiply by 1024 + total, used, free = (int(p) * 1024 for p in parts[1:4]) + return total, used, free + @retry_before_failing() def test_proc_memory_maps(self): out = sh('procstat -v %s' % self.pid) @@ -345,36 +358,19 @@ def test_cpu_stats_syscalls(self): # sysctl('vm.stats.sys.v_trap'), delta=1000) # --- swap memory - @staticmethod - def parse_swapinfo(): - - # the last line is always the total - output = sh("swapinfo -k").splitlines()[-1] - parts = re.split(r'\s+', output) - - if not parts: - raise ValueError("Can't parse swapinfo: %s" % output) - - # the size is in 1k units, so multiply by 1024 - total, used, free = (int(p) * 1024 for p in parts[1:4]) - - return total, used, free def test_swapmem_free(self): total, used, free = self.parse_swapinfo() - self.assertAlmostEqual( psutil.swap_memory().free, free, delta=MEMORY_TOLERANCE) def test_swapmem_used(self): total, used, free = self.parse_swapinfo() - self.assertAlmostEqual( psutil.swap_memory().used, used, delta=MEMORY_TOLERANCE) def test_swapmem_total(self): total, used, free = self.parse_swapinfo() - self.assertAlmostEqual( psutil.swap_memory().total, total, delta=MEMORY_TOLERANCE) From cb43f64ec4a81b1ffabc7c20f67f71b8d6c58e98 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 11 Apr 2018 09:30:39 +0200 Subject: [PATCH 056/182] have safe_rmpath() on windows retry for 1 sec in case of failure as a workaround for orphaned open handles --- psutil/tests/__init__.py | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index dcdbd4fa8..80404a332 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -357,7 +357,7 @@ def create_proc_children_pair(): subp = pyrun(s) child1 = psutil.Process(subp.pid) data = wait_for_file(_TESTFN2, delete=False, empty=False) - os.remove(_TESTFN2) + safe_rmpath(_TESTFN2) child2_pid = int(data) _pids_started.add(child2_pid) child2 = psutil.Process(child2_pid) @@ -663,7 +663,7 @@ def wait_for_file(fname, delete=True, empty=False): if not empty: assert data if delete: - os.remove(fname) + safe_rmpath(fname) return data @@ -685,12 +685,34 @@ def call_until(fun, expr): def safe_rmpath(path): "Convenience function for removing temporary test files or dirs" + def retry_fun(fun): + # On Windows it could happen that the file or directory has + # open handles or references preventing the delete operation + # to succeed immediately, so we retry for a while. See: + # https://bugs.python.org/issue33240 + stop_at = time.time() + 1 + while time.time() < stop_at: + try: + return fun() + except WindowsError as _: + err = _ + if err.errno != errno.ENOENT: + raise + else: + warn("ignoring %s" % (str(err))) + time.sleep(0.01) + raise err + try: st = os.stat(path) if stat.S_ISDIR(st.st_mode): - os.rmdir(path) + fun = functools.partial(shutil.rmtree, path) + else: + fun = functools.partial(os.remove, path) + if POSIX: + fun() else: - os.remove(path) + retry_fun(fun) except OSError as err: if err.errno != errno.ENOENT: raise From fa54c8ae4a384394efe867681a2c5d40a0d50de0 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 12 Apr 2018 07:50:46 +0200 Subject: [PATCH 057/182] adjust win deps --- appveyor.yml | 5 +---- scripts/internal/winmake.py | 12 ++++++++---- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 1bacc0651..a0ca6a0ed 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -71,14 +71,11 @@ install: # - ps: (new-object net.webclient).DownloadFile('https://raw.github.com/pypa/pip/master/contrib/get-pip.py', 'C:/get-pip.py') - "%WITH_COMPILER% %PYTHON%/python.exe -m pip --version" - "%WITH_COMPILER% %PYTHON%/python.exe -m pip install --upgrade --user setuptools pip" - - "%WITH_COMPILER% %PYTHON%/python.exe -m pip install --upgrade --user unittest2 ipaddress pypiwin32==219 wmi wheel" + - "%WITH_COMPILER% %PYTHON%/python.exe scripts/internal/winmake.py setup-dev-env" - "%WITH_COMPILER% %PYTHON%/python.exe -m pip freeze" - - "%WITH_COMPILER% %PYTHON%/python.exe scripts/internal/winmake.py clean" - "%WITH_COMPILER% %PYTHON%/python.exe setup.py build" - "%WITH_COMPILER% %PYTHON%/python.exe setup.py build build_ext -i" - "%WITH_COMPILER% %PYTHON%/python.exe setup.py develop" - # 1.0.1 is the latest release supporting python 2.6 - - "%WITH_COMPILER% %PYTHON%/Scripts/pip.exe install mock==1.0.1" build: off diff --git a/scripts/internal/winmake.py b/scripts/internal/winmake.py index 575437508..f12a40cf5 100755 --- a/scripts/internal/winmake.py +++ b/scripts/internal/winmake.py @@ -33,20 +33,24 @@ DEPS = [ "coverage", "flake8", - "ipaddress", - "mock", "nose", "pdbpp", "perf", "pip", - "pypiwin32", + "pypiwin32==219" if sys.version_info[:2] <= (3, 4) else "pypiwin32", "pyreadline", "setuptools", - "unittest2", "wheel", "wmi", "requests" ] +if sys.version_info[:2] <= (2, 6): + DEPS.append('unittest2') +if sys.version_info[:2] <= (2, 7): + DEPS.append('mock') +if sys.version_info[:2] <= (3, 2): + DEPS.append('ipaddress') + _cmds = {} if PY3: basestring = str From 540c71af92bb3ca60a2da63e4392f941087962ba Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 12 Apr 2018 10:29:02 +0200 Subject: [PATCH 058/182] another appveyor adjustment --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index a0ca6a0ed..8b73375e8 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -71,11 +71,11 @@ install: # - ps: (new-object net.webclient).DownloadFile('https://raw.github.com/pypa/pip/master/contrib/get-pip.py', 'C:/get-pip.py') - "%WITH_COMPILER% %PYTHON%/python.exe -m pip --version" - "%WITH_COMPILER% %PYTHON%/python.exe -m pip install --upgrade --user setuptools pip" - - "%WITH_COMPILER% %PYTHON%/python.exe scripts/internal/winmake.py setup-dev-env" - "%WITH_COMPILER% %PYTHON%/python.exe -m pip freeze" - "%WITH_COMPILER% %PYTHON%/python.exe setup.py build" - "%WITH_COMPILER% %PYTHON%/python.exe setup.py build build_ext -i" - "%WITH_COMPILER% %PYTHON%/python.exe setup.py develop" + - "%WITH_COMPILER% %PYTHON%/python.exe -m pip install --upgrade --user unittest2 ipaddress pypiwin32 wmi wheel" build: off From 8a5ad9adaa758598156d1a0c701969dfd3814d36 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 12 Apr 2018 10:30:37 +0200 Subject: [PATCH 059/182] and another one --- appveyor.yml | 2 +- scripts/internal/winmake.py | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 8b73375e8..f39053ade 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -75,7 +75,7 @@ install: - "%WITH_COMPILER% %PYTHON%/python.exe setup.py build" - "%WITH_COMPILER% %PYTHON%/python.exe setup.py build build_ext -i" - "%WITH_COMPILER% %PYTHON%/python.exe setup.py develop" - - "%WITH_COMPILER% %PYTHON%/python.exe -m pip install --upgrade --user unittest2 ipaddress pypiwin32 wmi wheel" + - "%WITH_COMPILER% %PYTHON%/python.exe scripts/internal/winmake.py setup-dev-env" build: off diff --git a/scripts/internal/winmake.py b/scripts/internal/winmake.py index f12a40cf5..bd518f092 100755 --- a/scripts/internal/winmake.py +++ b/scripts/internal/winmake.py @@ -24,7 +24,11 @@ import tempfile -PYTHON = os.getenv('PYTHON', sys.executable) +APPVEYOR = bool(os.environ.get('APPVEYOR')) +if APPVEYOR: + PYTHON = sys.executable +else: + PYTHON = os.getenv('PYTHON', sys.executable) TSCRIPT = os.getenv('TSCRIPT', 'psutil\\tests\\__main__.py') GET_PIP_URL = "https://bootstrap.pypa.io/get-pip.py" PY3 = sys.version_info[0] == 3 From b2bbd272f6b183ddc7c119d39ed5c969b399239f Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 12 Apr 2018 10:51:44 +0200 Subject: [PATCH 060/182] remove incorrect test assumption that proc cpu percent on windows is supposed to be <= 100, see https://ci.appveyor.com/project/giampaolo/psutil/build/1477/job/w1e0u92xrgg91ye3 --- psutil/tests/test_process.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index 3411114af..6f58be6d2 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -238,10 +238,6 @@ def test_cpu_percent(self): percent = p.cpu_percent(interval=None) self.assertIsInstance(percent, float) self.assertGreaterEqual(percent, 0.0) - if not POSIX: - self.assertLessEqual(percent, 100.0) - else: - self.assertGreaterEqual(percent, 0.0) with self.assertRaises(ValueError): p.cpu_percent(interval=-1) From 1ab4b3a1f416eda6c42414a7fbc2dd716e7e42f9 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 13 Apr 2018 04:16:16 +0200 Subject: [PATCH 061/182] adust winmake individual tests --- scripts/internal/winmake.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/scripts/internal/winmake.py b/scripts/internal/winmake.py index bd518f092..ff3a64891 100755 --- a/scripts/internal/winmake.py +++ b/scripts/internal/winmake.py @@ -365,7 +365,7 @@ def test_process(): """Run process tests""" install() test_setup() - sh("%s -m unittest -v psutil.tests.test_process" % PYTHON) + sh("%s psutil\\tests\\test_process.py" % PYTHON) @cmd @@ -373,7 +373,7 @@ def test_system(): """Run system tests""" install() test_setup() - sh("%s -m unittest -v psutil.tests.test_system" % PYTHON) + sh("%s psutil\\tests\\test_system.py" % PYTHON) @cmd @@ -381,7 +381,7 @@ def test_platform(): """Run windows only tests""" install() test_setup() - sh("%s -m unittest -v psutil.tests.test_windows" % PYTHON) + sh("%s psutil\\tests\\test_windows.py" % PYTHON) @cmd @@ -389,7 +389,7 @@ def test_misc(): """Run misc tests""" install() test_setup() - sh("%s -m unittest -v psutil.tests.test_misc" % PYTHON) + sh("%s psutil\\tests\\test_misc.py" % PYTHON) @cmd @@ -397,7 +397,7 @@ def test_unicode(): """Run unicode tests""" install() test_setup() - sh("%s -m unittest -v psutil.tests.test_unicode" % PYTHON) + sh("%s psutil\\tests\\test_unicode.py" % PYTHON) @cmd @@ -405,7 +405,7 @@ def test_connections(): """Run connections tests""" install() test_setup() - sh("%s -m unittest -v psutil.tests.test_connections" % PYTHON) + sh("%s psutil\\tests\\test_connections.py" % PYTHON) @cmd @@ -413,7 +413,7 @@ def test_contracts(): """Run contracts tests""" install() test_setup() - sh("%s -m unittest -v psutil.tests.test_contracts" % PYTHON) + sh("%s psutil\\tests\\test_contracts.py" % PYTHON) @cmd From ae7e4b4a9eebd3754cf0ee2a4ef0054a10151310 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 13 Apr 2018 04:26:21 +0200 Subject: [PATCH 062/182] fix some compilation warns on win --- psutil/_psutil_windows.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/psutil/_psutil_windows.c b/psutil/_psutil_windows.c index bb993dbd4..89d192697 100644 --- a/psutil/_psutil_windows.c +++ b/psutil/_psutil_windows.c @@ -855,7 +855,8 @@ psutil_proc_memory_uss(PyObject *self, PyObject *args) goto done; } - info_array_size = tmp_size + (entries * sizeof(PSAPI_WORKING_SET_BLOCK)); + info_array_size = tmp_size + \ + ((DWORD)entries * sizeof(PSAPI_WORKING_SET_BLOCK)); info_array = (PSAPI_WORKING_SET_INFORMATION*)malloc(info_array_size); if (!info_array) { PyErr_NoMemory(); @@ -2381,7 +2382,7 @@ psutil_disk_io_counters(PyObject *self, PyObject *args) { int devNum; int i; size_t ioctrlSize; - BOOL WINAPI ret; + BOOL ret; PyObject *py_retdict = PyDict_New(); PyObject *py_tuple = NULL; From 1e63ab355d592fefd77c819fb642fe6d7963f295 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 13 Apr 2018 04:31:51 +0200 Subject: [PATCH 063/182] fix some compilation warns on win --- psutil/_psutil_windows.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/psutil/_psutil_windows.c b/psutil/_psutil_windows.c index 89d192697..d39afb218 100644 --- a/psutil/_psutil_windows.c +++ b/psutil/_psutil_windows.c @@ -2958,7 +2958,7 @@ psutil_proc_memory_maps(PyObject *self, PyObject *args) { DWORD pid; HANDLE hProcess = NULL; PVOID baseAddress; - PVOID previousAllocationBase; + ULONGLONG previousAllocationBase; WCHAR mappedFileName[MAX_PATH]; SYSTEM_INFO system_info; LPVOID maxAddr; From 34e98b6e2a5739f9e633436e8cd61f3246c1091e Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 13 Apr 2018 04:52:55 -0400 Subject: [PATCH 064/182] 771 Windows CPU count (#1257) * use GetLogicalProcessorInformation() to get logical cpu_count() * return None if cpu_count() is undetermined + add mock test * style * factor out logical CPU num fun * remove unused code * psutil_get_num_cpus(): provide an option to fail on err * add comments * reuse get_num_cpus() function * error out if get_num_cpus() fail * use GetLogicalProcessorInformationEx to get phys CPU num * on win vista/xp just return None for phys CPU count * rename vars * fix C compiler warnings + remove mingw workarounds * return None if phys cpu count cant' be determined; update HISTORY * update HISTORY * update doc * add WMI tests * refactor tests * print debug msg for cpu phys returning None on win < 7 * try to fix win test * appveyor debug * fix typo * adjust appveyor 64 bit versions * debug msg * fix for loop * re-enable python versions * (maybe) finally fix GetLogicalProcessorInformationEx return len --- HISTORY.rst | 11 ++ MANIFEST.in | 1 - appveyor.yml | 18 +--- docs/index.rst | 17 +-- psutil/_psutil_common.c | 2 +- psutil/_psutil_windows.c | 197 +++++++++++++++++++++++------------ psutil/_pswindows.py | 2 +- psutil/arch/windows/glpi.h | 41 -------- psutil/tests/test_system.py | 7 +- psutil/tests/test_windows.py | 48 ++++++--- 10 files changed, 195 insertions(+), 149 deletions(-) delete mode 100644 psutil/arch/windows/glpi.h diff --git a/HISTORY.rst b/HISTORY.rst index 65cad58e0..b5fad59c5 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -15,6 +15,12 @@ XXXX-XX-XX - 694_: [SunOS] cmdline() could be truncated at the 15th character when reading it from /proc. An extra effort is made by reading it from process address space first. (patch by Georg Sauthoff) +- 771_: [Windows] cpu_count() (both logical and physical) return a wrong + (smaller) number on systems using process groups (> 64 cores). +- 771_: [Windows] cpu_times(percpu=True) return fewer CPUs on systems using + process groups (> 64 cores). +- 771_: [Windows] cpu_stats() and cpu_freq() may return incorrect results on + systems using process groups (> 64 cores). - 1193_: [SunOS] Return uid/gid from /proc/pid/psinfo if there aren't enough permissions for /proc/pid/cred. (patch by Georg Sauthoff) - 1194_: [SunOS] Return nice value from psinfo as getpriority() doesn't @@ -38,6 +44,11 @@ XXXX-XX-XX - 1255_: [FreeBSD] swap_memory() stats were erroneously represented in KB. (patch by Denis Krienbühl) +**Backward compatibility** + +- 771_: [Windows] cpu_count(logical=False) on Windows XP and Vista is no + longer supported and returns None. + 5.4.3 ===== diff --git a/MANIFEST.in b/MANIFEST.in index 87d9bebb4..85d1f21ec 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -67,7 +67,6 @@ include psutil/arch/solaris/environ.c include psutil/arch/solaris/environ.h include psutil/arch/solaris/v10/ifaddrs.c include psutil/arch/solaris/v10/ifaddrs.h -include psutil/arch/windows/glpi.h include psutil/arch/windows/inet_ntop.c include psutil/arch/windows/inet_ntop.h include psutil/arch/windows/ntextapi.h diff --git a/appveyor.yml b/appveyor.yml index f39053ade..436faadbc 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -45,20 +45,13 @@ environment: - PYTHON: "C:\\Python35-x64" PYTHON_VERSION: "3.5.x" PYTHON_ARCH: "64" - ARCH: x86_64 - VS_VER: "2015" - INSTANCENAME: "SQL2012SP1" - PYTHON: "C:\\Python36-x64" PYTHON_VERSION: "3.6.x" PYTHON_ARCH: "64" - ARCH: x86_64 - VS_VER: "2015" - INSTANCENAME: "SQL2012SP1" # Also build on a Python version not pre-installed by Appveyor. # See: https://github.com/ogrisel/python-appveyor-demo/issues/10 - # - PYTHON: "C:\\Python266" # PYTHON_VERSION: "2.6.6" # PYTHON_ARCH: "32" @@ -71,20 +64,17 @@ install: # - ps: (new-object net.webclient).DownloadFile('https://raw.github.com/pypa/pip/master/contrib/get-pip.py', 'C:/get-pip.py') - "%WITH_COMPILER% %PYTHON%/python.exe -m pip --version" - "%WITH_COMPILER% %PYTHON%/python.exe -m pip install --upgrade --user setuptools pip" - - "%WITH_COMPILER% %PYTHON%/python.exe -m pip freeze" - - "%WITH_COMPILER% %PYTHON%/python.exe setup.py build" - - "%WITH_COMPILER% %PYTHON%/python.exe setup.py build build_ext -i" - - "%WITH_COMPILER% %PYTHON%/python.exe setup.py develop" - "%WITH_COMPILER% %PYTHON%/python.exe scripts/internal/winmake.py setup-dev-env" + - "%WITH_COMPILER% %PYTHON%/python.exe -m pip freeze" + - "%WITH_COMPILER% %PYTHON%/python.exe scripts/internal/winmake.py install" build: off test_script: - - "%WITH_COMPILER% %PYTHON%/python -V" - - "set PYTHONWARNINGS=all && set PSUTIL_TESTING=1 && set PSUTIL_DEBUG=1 && %WITH_COMPILER% %PYTHON%/python psutil/tests/__main__.py" + - "%WITH_COMPILER% %PYTHON%/python.exe scripts/internal/winmake.py test" after_test: - - "%WITH_COMPILER% %PYTHON%/python setup.py bdist_wheel" + - "%WITH_COMPILER% %PYTHON%/python.exe scripts/internal/winmake.py wheel" artifacts: - path: dist\* diff --git a/docs/index.rst b/docs/index.rst index 395fd688f..548f361f4 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -164,15 +164,11 @@ CPU Return the number of logical CPUs in the system (same as `os.cpu_count() `__ in Python 3.4) or ``None`` if undetermined. - This number may not be equivalent to the number of CPUs the current process - can actually use in case process CPU affinity has been changed or Linux - cgroups are being used. - The number of usable CPUs can be obtained with - ``len(psutil.Process().cpu_affinity())``. If *logical* is ``False`` return the number of physical cores only (hyper - thread CPUs are excluded). + thread CPUs are excluded) or ``None`` if undetermined. On OpenBSD and NetBSD ``psutil.cpu_count(logical=False)`` always return - ``None``. Example on a system having 2 physical hyper-thread CPU cores: + ``None``. + Example on a system having 2 physical hyper-thread CPU cores: >>> import psutil >>> psutil.cpu_count() @@ -180,7 +176,12 @@ CPU >>> psutil.cpu_count(logical=False) 2 - Example returning the number of CPUs usable by the current process: + Note that this number is not equivalent to the number of CPUs the current + process can actually use. + That can vary in case process CPU affinity has been changed, Linux cgroups + are being used or on Windows systems using processor groups or having more + than 64 CPUs. + The number of usable CPUs can be obtained with: >>> len(psutil.Process().cpu_affinity()) 1 diff --git a/psutil/_psutil_common.c b/psutil/_psutil_common.c index e08f011c2..49b91c826 100644 --- a/psutil/_psutil_common.c +++ b/psutil/_psutil_common.c @@ -85,7 +85,7 @@ void psutil_debug(const char* format, ...) { va_list argptr; va_start(argptr, format); - fprintf(stderr, "psutil-dubug> "); + fprintf(stderr, "psutil-debug> "); vfprintf(stderr, format, argptr); fprintf(stderr, "\n"); va_end(argptr); diff --git a/psutil/_psutil_windows.c b/psutil/_psutil_windows.c index d39afb218..ae2e538a4 100644 --- a/psutil/_psutil_windows.c +++ b/psutil/_psutil_windows.c @@ -38,10 +38,6 @@ #include "arch/windows/inet_ntop.h" #include "arch/windows/services.h" -#ifdef __MINGW32__ -#include "arch/windows/glpi.h" -#endif - /* * ============================================================================ @@ -63,8 +59,13 @@ Py_DECREF(_SOCK_STREAM);\ Py_DECREF(_SOCK_DGRAM); -typedef BOOL (WINAPI *LPFN_GLPI) - (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION, PDWORD); +#if (_WIN32_WINNT >= 0x0601) // Windows 7 +typedef BOOL (WINAPI *PFN_GETLOGICALPROCESSORINFORMATIONEX)( + LOGICAL_PROCESSOR_RELATIONSHIP relationship, + PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX Buffer, + PDWORD ReturnLength); +static PFN_GETLOGICALPROCESSORINFORMATIONEX _GetLogicalProcessorInformationEx; +#endif // Fix for mingw32, see: // https://github.com/giampaolo/psutil/issues/351#c2 @@ -195,6 +196,45 @@ psutil_get_nic_addresses() { } +/* + * Return the number of logical, active CPUs. Return 0 if undetermined. + * See discussion at: https://bugs.python.org/issue33166#msg314631 + */ +unsigned int +psutil_get_num_cpus(int fail_on_err) { + unsigned int ncpus = 0; + SYSTEM_INFO sysinfo; + static DWORD(CALLBACK *_GetActiveProcessorCount)(WORD) = NULL; + HINSTANCE hKernel32; + + // GetActiveProcessorCount is available only on 64 bit versions + // of Windows from Windows 7 onward. + // Windows Vista 64 bit and Windows XP doesn't have it. + hKernel32 = GetModuleHandleW(L"KERNEL32"); + _GetActiveProcessorCount = (void*)GetProcAddress( + hKernel32, "GetActiveProcessorCount"); + + if (_GetActiveProcessorCount != NULL) { + ncpus = _GetActiveProcessorCount(ALL_PROCESSOR_GROUPS); + if ((ncpus == 0) && (fail_on_err == 1)) { + PyErr_SetFromWindowsErr(0); + } + } + else { + psutil_debug("GetActiveProcessorCount() not available; " + "using GetNativeSystemInfo()"); + GetNativeSystemInfo(&sysinfo); + ncpus = (unsigned int)sysinfo.dwNumberOfProcessors; + if ((ncpus == 0) && (fail_on_err == 1)) { + PyErr_SetString( + PyExc_RuntimeError, + "GetNativeSystemInfo() failed to retrieve CPU count"); + } + } + return ncpus; +} + + /* * ============================================================================ * Public Python API @@ -553,55 +593,77 @@ psutil_proc_create_time(PyObject *self, PyObject *args) { } - /* - * Return the number of logical CPUs. + * Return the number of active, logical CPUs. */ static PyObject * psutil_cpu_count_logical(PyObject *self, PyObject *args) { - SYSTEM_INFO system_info; - system_info.dwNumberOfProcessors = 0; + unsigned int ncpus; - GetSystemInfo(&system_info); - if (system_info.dwNumberOfProcessors == 0) - Py_RETURN_NONE; // mimic os.cpu_count() + ncpus = psutil_get_num_cpus(0); + if (ncpus != 0) + return Py_BuildValue("I", ncpus); else - return Py_BuildValue("I", system_info.dwNumberOfProcessors); + Py_RETURN_NONE; // mimick os.cpu_count() } /* - * Return the number of physical CPU cores. + * Return the number of physical CPU cores (hyper-thread CPUs count + * is excluded). */ +#if (_WIN32_WINNT < 0x0601) // < Windows 7 (namely Vista and XP) +static PyObject * +psutil_cpu_count_phys(PyObject *self, PyObject *args) { + // Note: we may have used GetLogicalProcessorInformation() + // but I don't want to prolong support for Windows XP and Vista. + // On such old systems psutil will compile but this API will + // just return None. + psutil_debug("Win < 7; cpu_count_phys() forced to None"); + Py_RETURN_NONE; +} +#else // Windows >= 7 static PyObject * psutil_cpu_count_phys(PyObject *self, PyObject *args) { - LPFN_GLPI glpi; DWORD rc; - PSYSTEM_LOGICAL_PROCESSOR_INFORMATION buffer = NULL; - PSYSTEM_LOGICAL_PROCESSOR_INFORMATION ptr = NULL; + PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX buffer = NULL; + PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX ptr = NULL; DWORD length = 0; DWORD offset = 0; - int ncpus = 0; - - glpi = (LPFN_GLPI)GetProcAddress(GetModuleHandle(TEXT("kernel32")), - "GetLogicalProcessorInformation"); - if (glpi == NULL) + DWORD ncpus = 0; + + // GetLogicalProcessorInformationEx() is available from Windows 7 + // onward. Differently from GetLogicalProcessorInformation() + // it supports process groups, meaning this is able to report more + // than 64 CPUs. See: + // https://bugs.python.org/issue33166 + _GetLogicalProcessorInformationEx = \ + (PFN_GETLOGICALPROCESSORINFORMATIONEX)GetProcAddress( + GetModuleHandle(TEXT("kernel32")), + "GetLogicalProcessorInformationEx"); + if (_GetLogicalProcessorInformationEx == NULL) { + psutil_debug("failed loading GetLogicalProcessorInformationEx()"); goto return_none; + } while (1) { - rc = glpi(buffer, &length); + rc = _GetLogicalProcessorInformationEx( + RelationAll, buffer, &length); if (rc == FALSE) { if (GetLastError() == ERROR_INSUFFICIENT_BUFFER) { - if (buffer) + if (buffer) { free(buffer); - buffer = (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION)malloc( - length); + } + buffer = \ + (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX)malloc(length); if (NULL == buffer) { PyErr_NoMemory(); return NULL; } } else { + psutil_debug("GetLogicalProcessorInformationEx() returned ", + GetLastError()); goto return_none; } } @@ -611,25 +673,30 @@ psutil_cpu_count_phys(PyObject *self, PyObject *args) { } ptr = buffer; - while (offset + sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION) <= length) { - if (ptr->Relationship == RelationProcessorCore) + while (ptr->Size > 0 && offset + ptr->Size <= length) { + if (ptr->Relationship == RelationProcessorCore) { ncpus += 1; - offset += sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION); - ptr++; + } + offset += ptr->Size; + ptr = (SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX*)\ + (((char*)ptr) + ptr->Size); } free(buffer); - if (ncpus == 0) - goto return_none; - else - return Py_BuildValue("i", ncpus); + if (ncpus != 0) { + return Py_BuildValue("I", ncpus); + } + else { + psutil_debug("GetLogicalProcessorInformationEx() count was 0"); + Py_RETURN_NONE; // mimick os.cpu_count() + } return_none: - // mimic os.cpu_count() if (buffer != NULL) free(buffer); Py_RETURN_NONE; } +#endif /* @@ -957,8 +1024,8 @@ psutil_per_cpu_times(PyObject *self, PyObject *args) { double idle, kernel, systemt, user, interrupt, dpc; NTSTATUS status; _SYSTEM_PROCESSOR_PERFORMANCE_INFORMATION *sppi = NULL; - SYSTEM_INFO si; UINT i; + unsigned int ncpus; PyObject *py_tuple = NULL; PyObject *py_retlist = PyList_New(0); @@ -978,14 +1045,15 @@ psutil_per_cpu_times(PyObject *self, PyObject *args) { goto error; } - // retrives number of processors - GetSystemInfo(&si); + // retrieves number of processors + ncpus = psutil_get_num_cpus(1); + if (ncpus == 0) + goto error; // allocates an array of _SYSTEM_PROCESSOR_PERFORMANCE_INFORMATION // structures, one per processor sppi = (_SYSTEM_PROCESSOR_PERFORMANCE_INFORMATION *) \ - malloc(si.dwNumberOfProcessors * \ - sizeof(_SYSTEM_PROCESSOR_PERFORMANCE_INFORMATION)); + malloc(ncpus * sizeof(_SYSTEM_PROCESSOR_PERFORMANCE_INFORMATION)); if (sppi == NULL) { PyErr_NoMemory(); goto error; @@ -995,8 +1063,7 @@ psutil_per_cpu_times(PyObject *self, PyObject *args) { status = NtQuerySystemInformation( SystemProcessorPerformanceInformation, sppi, - si.dwNumberOfProcessors * sizeof - (_SYSTEM_PROCESSOR_PERFORMANCE_INFORMATION), + ncpus * sizeof(_SYSTEM_PROCESSOR_PERFORMANCE_INFORMATION), NULL); if (status != 0) { PyErr_SetFromWindowsErr(0); @@ -1006,7 +1073,7 @@ psutil_per_cpu_times(PyObject *self, PyObject *args) { // computes system global times summing each // processor value idle = user = kernel = interrupt = dpc = 0; - for (i = 0; i < si.dwNumberOfProcessors; i++) { + for (i = 0; i < ncpus; i++) { py_tuple = NULL; user = (double)((HI_T * sppi[i].UserTime.HighPart) + (LO_T * sppi[i].UserTime.LowPart)); @@ -3403,7 +3470,7 @@ psutil_cpu_stats(PyObject *self, PyObject *args) { _SYSTEM_PERFORMANCE_INFORMATION *spi = NULL; _SYSTEM_PROCESSOR_PERFORMANCE_INFORMATION *sppi = NULL; _SYSTEM_INTERRUPT_INFORMATION *InterruptInformation = NULL; - SYSTEM_INFO si; + unsigned int ncpus; UINT i; ULONG64 dpcs = 0; ULONG interrupts = 0; @@ -3421,13 +3488,14 @@ psutil_cpu_stats(PyObject *self, PyObject *args) { goto error; } - // retrives number of processors - GetSystemInfo(&si); + // retrieves number of processors + ncpus = psutil_get_num_cpus(1); + if (ncpus == 0) + goto error; // get syscalls / ctx switches spi = (_SYSTEM_PERFORMANCE_INFORMATION *) \ - malloc(si.dwNumberOfProcessors * \ - sizeof(_SYSTEM_PERFORMANCE_INFORMATION)); + malloc(ncpus * sizeof(_SYSTEM_PERFORMANCE_INFORMATION)); if (spi == NULL) { PyErr_NoMemory(); goto error; @@ -3435,7 +3503,7 @@ psutil_cpu_stats(PyObject *self, PyObject *args) { status = NtQuerySystemInformation( SystemPerformanceInformation, spi, - si.dwNumberOfProcessors * sizeof(_SYSTEM_PERFORMANCE_INFORMATION), + ncpus * sizeof(_SYSTEM_PERFORMANCE_INFORMATION), NULL); if (status != 0) { PyErr_SetFromWindowsErr(0); @@ -3444,8 +3512,7 @@ psutil_cpu_stats(PyObject *self, PyObject *args) { // get DPCs InterruptInformation = \ - malloc(sizeof(_SYSTEM_INTERRUPT_INFORMATION) * - si.dwNumberOfProcessors); + malloc(sizeof(_SYSTEM_INTERRUPT_INFORMATION) * ncpus); if (InterruptInformation == NULL) { PyErr_NoMemory(); goto error; @@ -3454,20 +3521,19 @@ psutil_cpu_stats(PyObject *self, PyObject *args) { status = NtQuerySystemInformation( SystemInterruptInformation, InterruptInformation, - si.dwNumberOfProcessors * sizeof(SYSTEM_INTERRUPT_INFORMATION), + ncpus * sizeof(SYSTEM_INTERRUPT_INFORMATION), NULL); if (status != 0) { PyErr_SetFromWindowsErr(0); goto error; } - for (i = 0; i < si.dwNumberOfProcessors; i++) { + for (i = 0; i < ncpus; i++) { dpcs += InterruptInformation[i].DpcCount; } // get interrupts sppi = (_SYSTEM_PROCESSOR_PERFORMANCE_INFORMATION *) \ - malloc(si.dwNumberOfProcessors * \ - sizeof(_SYSTEM_PROCESSOR_PERFORMANCE_INFORMATION)); + malloc(ncpus * sizeof(_SYSTEM_PROCESSOR_PERFORMANCE_INFORMATION)); if (sppi == NULL) { PyErr_NoMemory(); goto error; @@ -3476,15 +3542,14 @@ psutil_cpu_stats(PyObject *self, PyObject *args) { status = NtQuerySystemInformation( SystemProcessorPerformanceInformation, sppi, - si.dwNumberOfProcessors * sizeof - (_SYSTEM_PROCESSOR_PERFORMANCE_INFORMATION), + ncpus * sizeof(_SYSTEM_PROCESSOR_PERFORMANCE_INFORMATION), NULL); if (status != 0) { PyErr_SetFromWindowsErr(0); goto error; } - for (i = 0; i < si.dwNumberOfProcessors; i++) { + for (i = 0; i < ncpus; i++) { interrupts += sppi[i].InterruptCount; } @@ -3525,19 +3590,15 @@ psutil_cpu_freq(PyObject *self, PyObject *args) { LPBYTE pBuffer = NULL; ULONG current; ULONG max; - unsigned int num_cpus; - SYSTEM_INFO system_info; - system_info.dwNumberOfProcessors = 0; + unsigned int ncpus; // Get the number of CPUs. - GetSystemInfo(&system_info); - if (system_info.dwNumberOfProcessors == 0) - num_cpus = 1; - else - num_cpus = system_info.dwNumberOfProcessors; + ncpus = psutil_get_num_cpus(1); + if (ncpus == 0) + return NULL; // Allocate size. - size = num_cpus * sizeof(PROCESSOR_POWER_INFORMATION); + size = ncpus * sizeof(PROCESSOR_POWER_INFORMATION); pBuffer = (BYTE*)LocalAlloc(LPTR, size); if (! pBuffer) return PyErr_SetFromWindowsErr(0); diff --git a/psutil/_pswindows.py b/psutil/_pswindows.py index ab727cba3..18651d6cf 100644 --- a/psutil/_pswindows.py +++ b/psutil/_pswindows.py @@ -288,7 +288,7 @@ def cpu_count_logical(): def cpu_count_physical(): - """Return the number of physical CPUs in the system.""" + """Return the number of physical CPU cores in the system.""" return cext.cpu_count_phys() diff --git a/psutil/arch/windows/glpi.h b/psutil/arch/windows/glpi.h deleted file mode 100644 index 6f9848373..000000000 --- a/psutil/arch/windows/glpi.h +++ /dev/null @@ -1,41 +0,0 @@ -// mingw headers are missing this - -typedef enum _LOGICAL_PROCESSOR_RELATIONSHIP { - RelationProcessorCore, - RelationNumaNode, - RelationCache, - RelationProcessorPackage, - RelationGroup, - RelationAll=0xffff -} LOGICAL_PROCESSOR_RELATIONSHIP; - -typedef enum _PROCESSOR_CACHE_TYPE { - CacheUnified,CacheInstruction,CacheData,CacheTrace -} PROCESSOR_CACHE_TYPE; - -typedef struct _CACHE_DESCRIPTOR { - BYTE Level; - BYTE Associativity; - WORD LineSize; - DWORD Size; - PROCESSOR_CACHE_TYPE Type; -} CACHE_DESCRIPTOR,*PCACHE_DESCRIPTOR; - -typedef struct _SYSTEM_LOGICAL_PROCESSOR_INFORMATION { - ULONG_PTR ProcessorMask; - LOGICAL_PROCESSOR_RELATIONSHIP Relationship; - union { - struct { - BYTE Flags; - } ProcessorCore; - struct { - DWORD NodeNumber; - } NumaNode; - CACHE_DESCRIPTOR Cache; - ULONGLONG Reserved[2]; - }; -} SYSTEM_LOGICAL_PROCESSOR_INFORMATION,*PSYSTEM_LOGICAL_PROCESSOR_INFORMATION; - -WINBASEAPI WINBOOL WINAPI -GetLogicalProcessorInformation(PSYSTEM_LOGICAL_PROCESSOR_INFORMATION Buffer, - PDWORD ReturnedLength); \ No newline at end of file diff --git a/psutil/tests/test_system.py b/psutil/tests/test_system.py index 20b132a91..6081ea5a2 100755 --- a/psutil/tests/test_system.py +++ b/psutil/tests/test_system.py @@ -268,8 +268,11 @@ def test_cpu_count(self): if "physical id" not in cpuinfo_data: raise unittest.SkipTest("cpuinfo doesn't include physical id") physical = psutil.cpu_count(logical=False) - self.assertGreaterEqual(physical, 1) - self.assertGreaterEqual(logical, physical) + if WINDOWS and sys.getwindowsversion()[:2] <= (6, 1): # <= Vista + self.assertIsNone(physical) + else: + self.assertGreaterEqual(physical, 1) + self.assertGreaterEqual(logical, physical) def test_cpu_count_none(self): # https://github.com/giampaolo/psutil/issues/1085 diff --git a/psutil/tests/test_windows.py b/psutil/tests/test_windows.py index 32c46f67a..ffa763d09 100755 --- a/psutil/tests/test_windows.py +++ b/psutil/tests/test_windows.py @@ -69,35 +69,57 @@ def wrapper(self, *args, **kwargs): @unittest.skipIf(not WINDOWS, "WINDOWS only") -class TestSystemAPIs(unittest.TestCase): - - def test_nic_names(self): - out = sh('ipconfig /all') - nics = psutil.net_io_counters(pernic=True).keys() - for nic in nics: - if "pseudo-interface" in nic.replace(' ', '-').lower(): - continue - if nic not in out: - self.fail( - "%r nic wasn't found in 'ipconfig /all' output" % nic) +class TestCpuAPIs(unittest.TestCase): @unittest.skipIf('NUMBER_OF_PROCESSORS' not in os.environ, 'NUMBER_OF_PROCESSORS env var is not available') - def test_cpu_count(self): + def test_cpu_count_vs_NUMBER_OF_PROCESSORS(self): + # Will likely fail on many-cores systems: + # https://stackoverflow.com/questions/31209256 num_cpus = int(os.environ['NUMBER_OF_PROCESSORS']) self.assertEqual(num_cpus, psutil.cpu_count()) - def test_cpu_count_2(self): + def test_cpu_count_vs_GetSystemInfo(self): + # Will likely fail on many-cores systems: + # https://stackoverflow.com/questions/31209256 sys_value = win32api.GetSystemInfo()[5] psutil_value = psutil.cpu_count() self.assertEqual(sys_value, psutil_value) + def test_cpu_count_logical_vs_wmi(self): + w = wmi.WMI() + proc = w.Win32_Processor()[0] + self.assertEqual(psutil.cpu_count(), proc.NumberOfLogicalProcessors) + + def test_cpu_count_phys_vs_wmi(self): + w = wmi.WMI() + proc = w.Win32_Processor()[0] + self.assertEqual(psutil.cpu_count(logical=False), proc.NumberOfCores) + + def test_cpu_count_vs_cpu_times(self): + self.assertEqual(psutil.cpu_count(), + len(psutil.cpu_times(percpu=True))) + def test_cpu_freq(self): w = wmi.WMI() proc = w.Win32_Processor()[0] self.assertEqual(proc.CurrentClockSpeed, psutil.cpu_freq().current) self.assertEqual(proc.MaxClockSpeed, psutil.cpu_freq().max) + +@unittest.skipIf(not WINDOWS, "WINDOWS only") +class TestSystemAPIs(unittest.TestCase): + + def test_nic_names(self): + out = sh('ipconfig /all') + nics = psutil.net_io_counters(pernic=True).keys() + for nic in nics: + if "pseudo-interface" in nic.replace(' ', '-').lower(): + continue + if nic not in out: + self.fail( + "%r nic wasn't found in 'ipconfig /all' output" % nic) + def test_total_phymem(self): w = wmi.WMI().Win32_ComputerSystem()[0] self.assertEqual(int(w.TotalPhysicalMemory), From feded95354062d3cc36f71993791b32a1c87098c Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 13 Apr 2018 11:09:43 +0200 Subject: [PATCH 065/182] pre-release --- HISTORY.rst | 2 +- docs/index.rst | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index b5fad59c5..a2913b265 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -3,7 +3,7 @@ 5.4.4 ===== -XXXX-XX-XX +2018-04-13 **Enhancements** diff --git a/docs/index.rst b/docs/index.rst index 548f361f4..67a7c5673 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -2624,9 +2624,13 @@ take a look at the Timeline ======== +- 2018-04-13: + `5.4.4 `__ - + `what's new `__ - + `diff `__ - 2018-01-01: `5.4.3 `__ - - `what's new `__ - + `what's new `__ - `diff `__ - 2017-12-07: `5.4.2 `__ - From 91b0d9c05d5781d3cf6594f2a3660ee897be0345 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 13 Apr 2018 19:34:06 +0200 Subject: [PATCH 066/182] #1268: fix setup.py's extra_require --- HISTORY.rst | 10 ++++++++++ docs/index.rst | 4 ++++ psutil/__init__.py | 2 +- setup.py | 22 ++++++++++++++-------- 4 files changed, 29 insertions(+), 9 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index a2913b265..685a8718e 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,5 +1,15 @@ *Bug tracker at https://github.com/giampaolo/psutil/issues* +5.4.5 +===== + +2018-04-14 + +**Bug fixes** + +- 1268_: setup.py's extra_require parameter requires latest setuptools version, + breaking quite a lot of installations. + 5.4.4 ===== diff --git a/docs/index.rst b/docs/index.rst index 67a7c5673..c8f7be8f5 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -2624,6 +2624,10 @@ take a look at the Timeline ======== +- 2018-04-14: + `5.4.5 `__ - + `what's new `__ - + `diff `__ - 2018-04-13: `5.4.4 `__ - `what's new `__ - diff --git a/psutil/__init__.py b/psutil/__init__.py index 1b0f2c045..90e8b15e5 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -218,7 +218,7 @@ ] __all__.extend(_psplatform.__extra__all__) __author__ = "Giampaolo Rodola'" -__version__ = "5.4.4" +__version__ = "5.4.5" version_info = tuple([int(num) for num in __version__.split('.')]) AF_LINK = _psplatform.AF_LINK POWER_TIME_UNLIMITED = _common.POWER_TIME_UNLIMITED diff --git a/setup.py b/setup.py index 61056f5f1..d8db694eb 100755 --- a/setup.py +++ b/setup.py @@ -52,6 +52,18 @@ if POSIX: sources.append('psutil/_psutil_posix.c') +tests_require = [] +if sys.version_info[:2] <= (2, 6): + tests_require.append('unittest2') +if sys.version_info[:2] <= (2, 7): + tests_require.append('mock') +if sys.version_info[:2] <= (3, 2): + tests_require.append('ipaddress') + +extras_require = {} +if sys.version_info[:2] <= (3, 3): + extras_require.update(dict(enum='enum34')) + def get_version(): INIT = os.path.join(HERE, 'psutil/__init__.py') @@ -328,14 +340,8 @@ def main(): kwargs.update( python_requires=">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*", test_suite="psutil.tests.get_suite", - tests_require=[ - 'ipaddress; python_version < "3.3"', - 'mock; python_version < "3.3"', - 'unittest2; python_version < "2.7"', - ], - extras_require={ - 'enum': 'enum34; python_version < "3.4"', - }, + tests_require=tests_require, + extras_require=extras_require, zip_safe=False, ) setup(**kwargs) From 70b330221b87ff9c42d5160c57efea8fd3804398 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 18 Apr 2018 11:17:08 +0200 Subject: [PATCH 067/182] fix doc --- docs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index c8f7be8f5..31b53384c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -449,7 +449,7 @@ Disks Network ------- -.. function:: net_io_counters(pernic=False) +.. function:: net_io_counters(pernic=False, nowrap=True) Return system-wide network I/O statistics as a named tuple including the following attributes: From 489422b9cccf55100a19e96135d05f8e01d48daa Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 25 Apr 2018 06:15:58 +0200 Subject: [PATCH 068/182] #1270: migrate URLs to new pypi site --- INSTALL.rst | 6 +- Makefile | 2 +- README.rst | 8 +- docs/index.rst | 140 ++++++++++++++--------------- scripts/internal/print_announce.py | 2 +- scripts/internal/print_timeline.py | 2 +- 6 files changed, 80 insertions(+), 80 deletions(-) diff --git a/INSTALL.rst b/INSTALL.rst index 9e5defb42..4ceb0c19b 100644 --- a/INSTALL.rst +++ b/INSTALL.rst @@ -76,15 +76,15 @@ Windows The easiest way to install psutil on Windows is to just use the pre-compiled exe/wheel installers hosted on -`PYPI `__ via pip: +`PYPI `__ via pip: .. code-block:: bat C:\Python27\python.exe -m pip install psutil If you want to compile psutil from sources you'll need **Visual Studio** -(Mingw32 is no longer supported), which really is a mess. -The VS versions are the onle listed below. +(Mingw32 is no longer supported), which really is a mess. +The VS versions are the onle listed below. This `blog post `__ provides numerous info on how to properly set up VS (good luck with that). diff --git a/Makefile b/Makefile index f7b4cd7f5..5693bb953 100644 --- a/Makefile +++ b/Makefile @@ -206,7 +206,7 @@ win-download-wheels: ## Download wheels hosted on appveyor. # --- upload -upload-src: ## Upload source tarball on https://pypi.python.org/pypi/psutil. +upload-src: ## Upload source tarball on https://pypi.org/project/psutil/ ${MAKE} sdist $(PYTHON) setup.py sdist upload diff --git a/README.rst b/README.rst index 73205270c..3f2a65203 100644 --- a/README.rst +++ b/README.rst @@ -15,7 +15,7 @@ :alt: Documentation Status .. image:: https://img.shields.io/pypi/v/psutil.svg?label=pypi - :target: https://pypi.python.org/pypi/psutil/ + :target: https://pypi.org/project/psutil :alt: Latest version .. image:: https://img.shields.io/github/stars/giampaolo/psutil.svg @@ -23,7 +23,7 @@ :alt: Github stars .. image:: https://img.shields.io/pypi/l/psutil.svg - :target: https://pypi.python.org/pypi/psutil/ + :target: https://pypi.org/project/psutil :alt: License =========== @@ -33,7 +33,7 @@ Quick links - `Home page `_ - `Install `_ - `Documentation `_ -- `Download `_ +- `Download `_ - `Forum `_ - `StackOverflow `_ - `Blog `_ @@ -86,7 +86,7 @@ Projects using psutil At the time of writing psutil has roughly `2.9 milion downloads `__ per month and there are over -`7000 open source projects `__ +`8000 open source projects `__ on github which depend from psutil. Here's some I find particularly interesting: diff --git a/docs/index.rst b/docs/index.rst index 31b53384c..866eaf319 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -12,7 +12,7 @@ Quick links - `Install `_ - `Blog `__ - `Forum `__ -- `Download `__ +- `Download `__ - `Development guide `_ - `What's new `__ @@ -39,7 +39,7 @@ psutil currently supports the following platforms: ...both **32-bit** and **64-bit** architectures, with Python versions from **2.6 to 3.6** (users of Python 2.4 and 2.5 may use -`2.1.3 `__ version). +`2.1.3 `__ version). `PyPy `__ is also known to work. The psutil documentation you're reading is distributed as a single HTML page. @@ -53,7 +53,7 @@ The easiest way to install psutil is via ``pip``:: On UNIX this requires a C compiler (e.g. gcc) installed. On Windows pip will automatically retrieve a pre-compiled wheel version from -`PYPI repository `__. +`PYPI repository `__. Alternatively, see more detailed `install `_ instructions. @@ -627,7 +627,7 @@ Network .. note:: if you're interested in others families (e.g. AF_BLUETOOTH) you can use - the more powerful `netifaces `__ + the more powerful `netifaces `__ extension. .. note:: @@ -2564,7 +2564,7 @@ FAQs * A: From Windows **Vista** onwards, both 32 and 64 bit versions. Latest binary (wheel / exe) release which supports Windows **2000**, **XP** and **2003 server** is - `psutil 3.4.2 `__. + `psutil 3.4.2 `__. On such old systems psutil is no longer tested or maintained, but it can still be compiled from sources (you'll need `Visual Studio <(https://github.com/giampaolo/psutil/blob/master/INSTALL.rst#windows>`__) and it should "work" (more or less). @@ -2573,7 +2573,7 @@ FAQs * Q: What Python versions are supported? * A: From 2.6 to 3.6, both 32 and 64 bit versions. Last version supporting - Python 2.4 and 2.5 is `psutil 2.1.3 `__. + Python 2.4 and 2.5 is `psutil 2.1.3 `__. PyPy is also known to work. ---- @@ -2625,258 +2625,258 @@ Timeline ======== - 2018-04-14: - `5.4.5 `__ - + `5.4.5 `__ - `what's new `__ - `diff `__ - 2018-04-13: - `5.4.4 `__ - + `5.4.4 `__ - `what's new `__ - `diff `__ - 2018-01-01: - `5.4.3 `__ - + `5.4.3 `__ - `what's new `__ - `diff `__ - 2017-12-07: - `5.4.2 `__ - + `5.4.2 `__ - `what's new `__ - `diff `__ - 2017-11-08: - `5.4.1 `__ - + `5.4.1 `__ - `what's new `__ - `diff `__ - 2017-10-12: - `5.4.0 `__ - + `5.4.0 `__ - `what's new `__ - `diff `__ - 2017-09-10: - `5.3.1 `__ - + `5.3.1 `__ - `what's new `__ - `diff `__ - 2017-09-01: - `5.3.0 `__ - + `5.3.0 `__ - `what's new `__ - `diff `__ - 2017-04-10: - `5.2.2 `__ - + `5.2.2 `__ - `what's new `__ - `diff `__ - 2017-03-24: - `5.2.1 `__ - + `5.2.1 `__ - `what's new `__ - `diff `__ - 2017-03-05: - `5.2.0 `__ - + `5.2.0 `__ - `what's new `__ - `diff `__ - 2017-02-07: - `5.1.3 `__ - + `5.1.3 `__ - `what's new `__ - `diff `__ - 2017-02-03: - `5.1.2 `__ - + `5.1.2 `__ - `what's new `__ - `diff `__ - 2017-02-03: - `5.1.1 `__ - + `5.1.1 `__ - `what's new `__ - `diff `__ - 2017-02-01: - `5.1.0 `__ - + `5.1.0 `__ - `what's new `__ - `diff `__ - 2016-12-21: - `5.0.1 `__ - + `5.0.1 `__ - `what's new `__ - `diff `__ - 2016-11-06: - `5.0.0 `__ - + `5.0.0 `__ - `what's new `__ - `diff `__ - 2016-10-05: - `4.4.2 `__ - + `4.4.2 `__ - `what's new `__ - `diff `__ - 2016-10-25: - `4.4.1 `__ - + `4.4.1 `__ - `what's new `__ - `diff `__ - 2016-10-23: - `4.4.0 `__ - + `4.4.0 `__ - `what's new `__ - `diff `__ - 2016-09-01: - `4.3.1 `__ - + `4.3.1 `__ - `what's new `__ - `diff `__ - 2016-06-18: - `4.3.0 `__ - + `4.3.0 `__ - `what's new `__ - `diff `__ - 2016-05-14: - `4.2.0 `__ - + `4.2.0 `__ - `what's new `__ - `diff `__ - 2016-03-12: - `4.1.0 `__ - + `4.1.0 `__ - `what's new `__ - `diff `__ - 2016-02-17: - `4.0.0 `__ - + `4.0.0 `__ - `what's new `__ - `diff `__ - 2016-01-20: - `3.4.2 `__ - + `3.4.2 `__ - `what's new `__ - `diff `__ - 2016-01-15: - `3.4.1 `__ - + `3.4.1 `__ - `what's new `__ - `diff `__ - 2015-11-25: - `3.3.0 `__ - + `3.3.0 `__ - `what's new `__ - `diff `__ - 2015-10-04: - `3.2.2 `__ - + `3.2.2 `__ - `what's new `__ - `diff `__ - 2015-09-03: - `3.2.1 `__ - + `3.2.1 `__ - `what's new `__ - `diff `__ - 2015-09-02: - `3.2.0 `__ - + `3.2.0 `__ - `what's new `__ - `diff `__ - 2015-07-15: - `3.1.1 `__ - + `3.1.1 `__ - `what's new `__ - `diff `__ - 2015-07-15: - `3.1.0 `__ - + `3.1.0 `__ - `what's new `__ - `diff `__ - 2015-06-18: - `3.0.1 `__ - + `3.0.1 `__ - `what's new `__ - `diff `__ - 2015-06-13: - `3.0.0 `__ - + `3.0.0 `__ - `what's new `__ - `diff `__ - 2015-02-02: - `2.2.1 `__ - + `2.2.1 `__ - `what's new `__ - `diff `__ - 2015-01-06: - `2.2.0 `__ - + `2.2.0 `__ - `what's new `__ - `diff `__ - 2014-09-26: - `2.1.3 `__ - + `2.1.3 `__ - `what's new `__ - `diff `__ - 2014-09-21: - `2.1.2 `__ - + `2.1.2 `__ - `what's new `__ - `diff `__ - 2014-04-30: - `2.1.1 `__ - + `2.1.1 `__ - `what's new `__ - `diff `__ - 2014-04-08: - `2.1.0 `__ - + `2.1.0 `__ - `what's new `__ - `diff `__ - 2014-03-10: - `2.0.0 `__ - + `2.0.0 `__ - `what's new `__ - `diff `__ - 2013-11-25: - `1.2.1 `__ - + `1.2.1 `__ - `what's new `__ - `diff `__ - 2013-11-20: - `1.2.0 `__ - + `1.2.0 `__ - `what's new `__ - `diff `__ - 2013-10-22: - `1.1.2 `__ - + `1.1.2 `__ - `what's new `__ - `diff `__ - 2013-10-08: - `1.1.1 `__ - + `1.1.1 `__ - `what's new `__ - `diff `__ - 2013-09-28: - `1.1.0 `__ - + `1.1.0 `__ - `what's new `__ - `diff `__ - 2013-07-12: - `1.0.1 `__ - + `1.0.1 `__ - `what's new `__ - `diff `__ - 2013-07-10: - `1.0.0 `__ - + `1.0.0 `__ - `what's new `__ - `diff `__ - 2013-05-03: - `0.7.1 `__ - + `0.7.1 `__ - `what's new `__ - `diff `__ - 2013-04-12: - `0.7.0 `__ - + `0.7.0 `__ - `what's new `__ - `diff `__ - 2012-08-16: - `0.6.1 `__ - + `0.6.1 `__ - `what's new `__ - `diff `__ - 2012-08-13: - `0.6.0 `__ - + `0.6.0 `__ - `what's new `__ - `diff `__ - 2012-06-29: - `0.5.1 `__ - + `0.5.1 `__ - `what's new `__ - `diff `__ - 2012-06-27: - `0.5.0 `__ - + `0.5.0 `__ - `what's new `__ - `diff `__ - 2011-12-14: - `0.4.1 `__ - + `0.4.1 `__ - `what's new `__ - `diff `__ - 2011-10-29: - `0.4.0 `__ - + `0.4.0 `__ - `what's new `__ - `diff `__ - 2011-07-08: - `0.3.0 `__ - + `0.3.0 `__ - `what's new `__ - `diff `__ - 2011-03-20: - `0.2.1 `__ - + `0.2.1 `__ - `what's new `__ - `diff `__ - 2010-11-13: - `0.2.0 `__ - + `0.2.0 `__ - `what's new `__ - `diff `__ - 2010-03-02: - `0.1.3 `__ - + `0.1.3 `__ - `what's new `__ - `diff `__ - 2009-05-06: - `0.1.2 `__ - + `0.1.2 `__ - `what's new `__ - `diff `__ - 2009-03-06: - `0.1.1 `__ - + `0.1.1 `__ - `what's new `__ - `diff `__ - 2009-01-27: - `0.1.0 `__ - + `0.1.0 `__ - `what's new `__ - `diff `__ diff --git a/scripts/internal/print_announce.py b/scripts/internal/print_announce.py index 1c2b9e113..018fb0928 100755 --- a/scripts/internal/print_announce.py +++ b/scripts/internal/print_announce.py @@ -20,7 +20,7 @@ PRJ_NAME = 'psutil' PRJ_URL_HOME = 'https://github.com/giampaolo/psutil' PRJ_URL_DOC = 'http://psutil.readthedocs.io' -PRJ_URL_DOWNLOAD = 'https://pypi.python.org/pypi/psutil' +PRJ_URL_DOWNLOAD = 'https://pypi.org/project/psutil/#files' PRJ_URL_WHATSNEW = \ 'https://github.com/giampaolo/psutil/blob/master/HISTORY.rst' diff --git a/scripts/internal/print_timeline.py b/scripts/internal/print_timeline.py index ffcb8fe85..4bfe76b33 100644 --- a/scripts/internal/print_timeline.py +++ b/scripts/internal/print_timeline.py @@ -13,7 +13,7 @@ entry = """\ - {date}: - `{ver} `__ - + `{ver} `__ - `what's new `__ - `diff `__""" # NOQA From abbe589de7e6109849924bf6172bd40132fdfc5c Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 4 May 2018 23:00:33 +0200 Subject: [PATCH 069/182] #1273 net_if_addr() namedtuple name has been renamed from "snic" to "snicaddr". --- HISTORY.rst | 10 ++++++++++ Makefile | 2 +- README.rst | 12 ++++++------ docs/index.rst | 12 ++++++------ psutil/__init__.py | 2 +- psutil/_common.py | 5 +++-- psutil/tests/__init__.py | 2 +- scripts/internal/{purge.py => purge_installation.py} | 0 8 files changed, 28 insertions(+), 17 deletions(-) rename scripts/internal/{purge.py => purge_installation.py} (100%) diff --git a/HISTORY.rst b/HISTORY.rst index 685a8718e..8f49025b7 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,5 +1,15 @@ *Bug tracker at https://github.com/giampaolo/psutil/issues* +5.4.6 +===== + +XXXX-XX-XX + +**Bug fixes** + +- 1273_: net_if_addr() namedtuple's name has been renamed from "snic" to + "snicaddr". + 5.4.5 ===== diff --git a/Makefile b/Makefile index 5693bb953..51047383f 100644 --- a/Makefile +++ b/Makefile @@ -77,7 +77,7 @@ install: ## Install this package as current user in "edit" mode. uninstall: ## Uninstall this package via pip. cd ..; $(PYTHON) -m pip uninstall -y -v psutil || true - $(PYTHON) scripts/internal/purge.py + $(PYTHON) scripts/internal/purge_installation.py install-pip: ## Install pip (no-op if already installed). $(PYTHON) -c \ diff --git a/README.rst b/README.rst index 3f2a65203..0852b1ce5 100644 --- a/README.rst +++ b/README.rst @@ -201,12 +201,12 @@ Network ...] >>> >>> psutil.net_if_addrs() - {'lo': [snic(family=, address='127.0.0.1', netmask='255.0.0.0', broadcast='127.0.0.1', ptp=None), - snic(family=, address='::1', netmask='ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff', broadcast=None, ptp=None), - snic(family=, address='00:00:00:00:00:00', netmask=None, broadcast='00:00:00:00:00:00', ptp=None)], - 'wlan0': [snic(family=, address='192.168.1.3', netmask='255.255.255.0', broadcast='192.168.1.255', ptp=None), - snic(family=, address='fe80::c685:8ff:fe45:641%wlan0', netmask='ffff:ffff:ffff:ffff::', broadcast=None, ptp=None), - snic(family=, address='c4:85:08:45:06:41', netmask=None, broadcast='ff:ff:ff:ff:ff:ff', ptp=None)]} + {'lo': [snicaddr(family=, address='127.0.0.1', netmask='255.0.0.0', broadcast='127.0.0.1', ptp=None), + snicaddr(family=, address='::1', netmask='ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff', broadcast=None, ptp=None), + snicaddr(family=, address='00:00:00:00:00:00', netmask=None, broadcast='00:00:00:00:00:00', ptp=None)], + 'wlan0': [snicaddr(family=, address='192.168.1.3', netmask='255.255.255.0', broadcast='192.168.1.255', ptp=None), + snicaddr(family=, address='fe80::c685:8ff:fe45:641%wlan0', netmask='ffff:ffff:ffff:ffff::', broadcast=None, ptp=None), + snicaddr(family=, address='c4:85:08:45:06:41', netmask=None, broadcast='ff:ff:ff:ff:ff:ff', ptp=None)]} >>> >>> psutil.net_if_stats() {'eth0': snicstats(isup=True, duplex=, speed=100, mtu=1500), diff --git a/docs/index.rst b/docs/index.rst index 866eaf319..6680a5620 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -613,12 +613,12 @@ Network >>> import psutil >>> psutil.net_if_addrs() - {'lo': [snic(family=, address='127.0.0.1', netmask='255.0.0.0', broadcast='127.0.0.1', ptp=None), - snic(family=, address='::1', netmask='ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff', broadcast=None, ptp=None), - snic(family=, address='00:00:00:00:00:00', netmask=None, broadcast='00:00:00:00:00:00', ptp=None)], - 'wlan0': [snic(family=, address='192.168.1.3', netmask='255.255.255.0', broadcast='192.168.1.255', ptp=None), - snic(family=, address='fe80::c685:8ff:fe45:641%wlan0', netmask='ffff:ffff:ffff:ffff::', broadcast=None, ptp=None), - snic(family=, address='c4:85:08:45:06:41', netmask=None, broadcast='ff:ff:ff:ff:ff:ff', ptp=None)]} + {'lo': [snicaddr(family=, address='127.0.0.1', netmask='255.0.0.0', broadcast='127.0.0.1', ptp=None), + snicaddr(family=, address='::1', netmask='ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff', broadcast=None, ptp=None), + snicaddr(family=, address='00:00:00:00:00:00', netmask=None, broadcast='00:00:00:00:00:00', ptp=None)], + 'wlan0': [snicaddr(family=, address='192.168.1.3', netmask='255.255.255.0', broadcast='192.168.1.255', ptp=None), + snicaddr(family=, address='fe80::c685:8ff:fe45:641%wlan0', netmask='ffff:ffff:ffff:ffff::', broadcast=None, ptp=None), + snicaddr(family=, address='c4:85:08:45:06:41', netmask=None, broadcast='ff:ff:ff:ff:ff:ff', ptp=None)]} >>> See also `nettop.py `__ diff --git a/psutil/__init__.py b/psutil/__init__.py index 90e8b15e5..2ecd3bddf 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -2155,7 +2155,7 @@ def net_if_addrs(): separator = ":" if POSIX else "-" while addr.count(separator) < 5: addr += "%s00" % separator - ret[name].append(_common.snic(fam, addr, mask, broadcast, ptp)) + ret[name].append(_common.snicaddr(fam, addr, mask, broadcast, ptp)) return dict(ret) diff --git a/psutil/_common.py b/psutil/_common.py index 05dbb4ce6..e65d39e3b 100644 --- a/psutil/_common.py +++ b/psutil/_common.py @@ -59,7 +59,7 @@ # named tuples 'pconn', 'pcputimes', 'pctxsw', 'pgids', 'pio', 'pionice', 'popenfile', 'pthread', 'puids', 'sconn', 'scpustats', 'sdiskio', 'sdiskpart', - 'sdiskusage', 'snetio', 'snic', 'snicstats', 'sswap', 'suser', + 'sdiskusage', 'snetio', 'snicaddr', 'snicstats', 'sswap', 'suser', # utility functions 'conn_tmap', 'deprecated_method', 'isfile_strict', 'memoize', 'parse_environ_block', 'path_exists_strict', 'usage_percent', @@ -182,7 +182,8 @@ class BatteryTime(enum.IntEnum): sconn = namedtuple('sconn', ['fd', 'family', 'type', 'laddr', 'raddr', 'status', 'pid']) # psutil.net_if_addrs() -snic = namedtuple('snic', ['family', 'address', 'netmask', 'broadcast', 'ptp']) +snicaddr = namedtuple('snicaddr', + ['family', 'address', 'netmask', 'broadcast', 'ptp']) # psutil.net_if_stats() snicstats = namedtuple('snicstats', ['isup', 'duplex', 'speed', 'mtu']) # psutil.cpu_stats() diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 80404a332..f11a6a028 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -125,7 +125,7 @@ # bytes tolerance for system-wide memory related tests MEMORY_TOLERANCE = 500 * 1024 # 500KB # the timeout used in functions which have to wait -GLOBAL_TIMEOUT = 3 +GLOBAL_TIMEOUT = 3 if TRAVIS or APPVEYOR else 0.5 # test output verbosity VERBOSITY = 1 if os.getenv('SILENT') or TOX else 2 # be more tolerant if we're on travis / appveyor in order to avoid diff --git a/scripts/internal/purge.py b/scripts/internal/purge_installation.py similarity index 100% rename from scripts/internal/purge.py rename to scripts/internal/purge_installation.py From 757f16b744a8b8d8ae442cf69732feff02e83ea8 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 4 May 2018 23:03:10 +0200 Subject: [PATCH 070/182] remove outdated linux test --- psutil/tests/test_linux.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index d4eaf2a14..9ea59b617 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -1408,20 +1408,6 @@ def test_emulate_energy_full_not_avail(self): "/sys/class/power_supply/BAT0/capacity", b"88"): self.assertEqual(psutil.sensors_battery().percent, 88) - def test_emulate_no_ac0_online(self): - # Emulate a case where /AC0/online file does not exist. - def path_exists_mock(name): - if name.startswith("/sys/class/power_supply/AC0/online"): - return False - else: - return orig_path_exists(name) - - orig_path_exists = os.path.exists - with mock.patch("psutil._pslinux.os.path.exists", - side_effect=path_exists_mock) as m: - psutil.sensors_battery() - assert m.called - def test_emulate_no_power(self): # Emulate a case where /AC0/online file nor /BAT0/status exist. with mock_open_exception( From 6ad33123e00383f9ea8a3e336783acec27c30448 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 4 May 2018 23:43:46 +0200 Subject: [PATCH 071/182] minor refactoring --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index d8db694eb..fffefd2f7 100755 --- a/setup.py +++ b/setup.py @@ -43,8 +43,6 @@ macros = [] if POSIX: macros.append(("PSUTIL_POSIX", 1)) -if WINDOWS: - macros.append(("PSUTIL_WINDOWS", 1)) if BSD: macros.append(("PSUTIL_BSD", 1)) @@ -117,6 +115,7 @@ def get_winver(): msg += "Visual Studio and may also (kind of) work though" warnings.warn(msg, UserWarning) + macros.append(("PSUTIL_WINDOWS", 1)) macros.extend([ # be nice to mingw, see: # http://www.mingw.org/wiki/Use_more_recent_defined_functions @@ -252,6 +251,7 @@ def get_ethtool_macro(): 'psutil/arch/aix/ifaddrs.c'], libraries=['perfstat'], define_macros=macros) + else: sys.exit('platform %s is not supported' % sys.platform) From 1b5f8d55d22167d640eeb52d0b82f69dec83d210 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 8 May 2018 18:13:40 +0200 Subject: [PATCH 072/182] fix #1274 / Process.children / Linux: do not swallow AccessDenied --- HISTORY.rst | 2 ++ psutil/__init__.py | 10 +++------- psutil/_pslinux.py | 5 ++--- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 8f49025b7..f69c5fb63 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -9,6 +9,8 @@ XXXX-XX-XX - 1273_: net_if_addr() namedtuple's name has been renamed from "snic" to "snicaddr". +- 1274_: [Linux] there was a small chance Process.children() may swallow + AccessDenied exceptions. 5.4.5 ===== diff --git a/psutil/__init__.py b/psutil/__init__.py index 2ecd3bddf..5e9a7fb6c 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -218,7 +218,7 @@ ] __all__.extend(_psplatform.__extra__all__) __author__ = "Giampaolo Rodola'" -__version__ = "5.4.5" +__version__ = "5.4.6" version_info = tuple([int(num) for num in __version__.split('.')]) AF_LINK = _psplatform.AF_LINK POWER_TIME_UNLIMITED = _common.POWER_TIME_UNLIMITED @@ -265,13 +265,9 @@ def _ppid_map(): ret = {} for pid in pids(): try: - proc = _psplatform.Process(pid) - ppid = proc.ppid() - except (NoSuchProcess, AccessDenied): - # Note: AccessDenied is unlikely to happen. + ret[pid] = _psplatform.Process(pid).ppid() + except (NoSuchProcess, ZombieProcess): pass - else: - ret[pid] = ppid return ret diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index 78c03d5c5..b197dba33 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -1380,9 +1380,8 @@ def ppid_map(): data = f.read() except EnvironmentError as err: # Note: we should be able to access /stat for all processes - # so we won't bump into EPERM, which is good. - if err.errno not in (errno.ENOENT, errno.ESRCH, - errno.EPERM, errno.EACCES): + # aka it's unlikely we'll bump into EPERM, which is good. + if err.errno not in (errno.ENOENT, errno.ESRCH): raise else: rpar = data.rfind(b')') From f4c6b02461cc89a7f9a98a164d346b4dfb72bee0 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 22 May 2018 17:48:38 +0200 Subject: [PATCH 073/182] fix freebsd compilation warning --- psutil/arch/freebsd/specific.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/psutil/arch/freebsd/specific.c b/psutil/arch/freebsd/specific.c index cf0b7df24..177ac8a6b 100644 --- a/psutil/arch/freebsd/specific.c +++ b/psutil/arch/freebsd/specific.c @@ -273,7 +273,6 @@ psutil_proc_exe(PyObject *self, PyObject *args) { int mib[4]; int ret; size_t size; - const char *encoding_errs; if (! PyArg_ParseTuple(args, "l", &pid)) return NULL; @@ -540,7 +539,6 @@ psutil_proc_cwd(PyObject *self, PyObject *args) { struct kinfo_file *freep = NULL; struct kinfo_file *kif; struct kinfo_proc kipp; - const char *encoding_errs; PyObject *py_path = NULL; int i, cnt; From ac9dccab6b038835b5d612f92cf4804ec2662c2e Mon Sep 17 00:00:00 2001 From: Jean-Luc Migot Date: Thu, 7 Jun 2018 17:13:40 +0200 Subject: [PATCH 074/182] Fix Windows crash on proc_username(), happens when WinAPI calls fail, leading to "goto error" (#1289) This will prevent deallocation of random memory not initialized (by the way you should stop using "goto"...) --- psutil/_psutil_windows.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/psutil/_psutil_windows.c b/psutil/_psutil_windows.c index ae2e538a4..ba898f60b 100644 --- a/psutil/_psutil_windows.c +++ b/psutil/_psutil_windows.c @@ -1405,9 +1405,9 @@ psutil_proc_username(PyObject *self, PyObject *args) { ULONG nameSize; ULONG domainNameSize; SID_NAME_USE nameUse; - PyObject *py_username; - PyObject *py_domain; - PyObject *py_tuple; + PyObject *py_username = NULL; + PyObject *py_domain = NULL; + PyObject *py_tuple = NULL; if (! PyArg_ParseTuple(args, "l", &pid)) return NULL; From e05f07722a994bc5c96c9f11eeaeec244dfd3516 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 7 Jun 2018 17:16:19 +0200 Subject: [PATCH 075/182] update HISTORY and CREDITS --- CREDITS | 4 ++++ HISTORY.rst | 2 ++ 2 files changed, 6 insertions(+) diff --git a/CREDITS b/CREDITS index 41061cdf7..f5f1aa864 100644 --- a/CREDITS +++ b/CREDITS @@ -535,3 +535,7 @@ I: 1239 N: Denis Krienbühl W: https://github.com/href I: 1260 + +N: Jean-Luc Migot +W: https://github.com/jmigot-tehtris +I: 1258, 1289 diff --git a/HISTORY.rst b/HISTORY.rst index f69c5fb63..6da633ef5 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -7,6 +7,8 @@ XXXX-XX-XX **Bug fixes** +- 1258_: [Windows] Process.username() may cause a segfault (Python interpreter + crash). (patch by Jean-Luc Migot) - 1273_: net_if_addr() namedtuple's name has been renamed from "snic" to "snicaddr". - 1274_: [Linux] there was a small chance Process.children() may swallow From 776d61024f061c47fbff5c765a619c1fd5701f5a Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 7 Jun 2018 17:39:07 +0200 Subject: [PATCH 076/182] pre-release --- HISTORY.rst | 2 +- MANIFEST.in | 2 +- docs/index.rst | 4 ++++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 6da633ef5..b07a0df81 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -3,7 +3,7 @@ 5.4.6 ===== -XXXX-XX-XX +2018-06-07 **Bug fixes** diff --git a/MANIFEST.in b/MANIFEST.in index 85d1f21ec..26d678fc3 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -109,7 +109,7 @@ include scripts/internal/download_exes.py include scripts/internal/generate_manifest.py include scripts/internal/print_announce.py include scripts/internal/print_timeline.py -include scripts/internal/purge.py +include scripts/internal/purge_installation.py include scripts/internal/winmake.py include scripts/iotop.py include scripts/killall.py diff --git a/docs/index.rst b/docs/index.rst index 6680a5620..aaf004cee 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -2624,6 +2624,10 @@ take a look at the Timeline ======== +- 2018-06-07: + `5.4.6 `__ - + `what's new `__ - + `diff `__ - 2018-04-14: `5.4.5 `__ - `what's new `__ - From c7bd2b6943402805bcaf3ac899c54278be0a75e8 Mon Sep 17 00:00:00 2001 From: Jon Dufresne Date: Sun, 10 Jun 2018 08:44:34 -0700 Subject: [PATCH 077/182] Update Python 2 docs URLs to Python 3 (#1293) The Python3 docs are more actively maintained and are the future of the Python project. --- docs/index.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index aaf004cee..bde3d453c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -993,11 +993,11 @@ Process class When accessing methods of this class always be prepared to catch :class:`NoSuchProcess`, :class:`ZombieProcess` and :class:`AccessDenied` exceptions. - `hash() `__ builtin can + `hash() `__ builtin can be used against instances of this class in order to identify a process univocally over time (the hash is determined by mixing process PID and creation time). As such it can also be used with - `set()s `__. + `set()s `__. .. note:: @@ -1761,7 +1761,7 @@ Process class ``'r+'`` and ``'a+'``. There's no distinction between files opened in bynary or text mode (``"b"`` or ``"t"``). - **flags** (*Linux*): the flags which were passed to the underlying - `os.open `__ C call + `os.open `__ C call when the file was opened (e.g. `os.O_RDONLY `__, `os.O_TRUNC `__, @@ -2600,7 +2600,7 @@ FAQs * Q: What about load average? * A: psutil does not expose any load average function as it's already available in python as - `os.getloadavg `__. + `os.getloadavg `__. Running tests ============= From 4b5745e94b7922df48e89e77d60218708d5baa1f Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 12 Jun 2018 15:56:14 +0200 Subject: [PATCH 078/182] import ssl only if necessary --- psutil/tests/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/psutil/tests/__main__.py b/psutil/tests/__main__.py index 62fe07428..36554a126 100755 --- a/psutil/tests/__main__.py +++ b/psutil/tests/__main__.py @@ -13,7 +13,6 @@ import contextlib import optparse import os -import ssl import sys import tempfile try: @@ -38,6 +37,7 @@ def install_pip(): try: import pip # NOQA except ImportError: + import ssl f = tempfile.NamedTemporaryFile(suffix='.py') with contextlib.closing(f): print("downloading %s to %s" % (GET_PIP_URL, f.name)) From 792499afbfdbad4b84ec5bc4acf2ed4e85310624 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 14 Jun 2018 15:33:20 -0700 Subject: [PATCH 079/182] OSX - wrapper around task_for_pid() (#1296) (OSX) wrapper around task_for_pid() fix #1181, fix #1209, fix #1291 --- HISTORY.rst | 10 +++++ psutil/_psosx.py | 8 ++-- psutil/_psutil_osx.c | 94 +++++++++++++++++++++++++++------------- psutil/tests/__main__.py | 2 +- 4 files changed, 79 insertions(+), 35 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index b07a0df81..5854fa9fd 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,5 +1,15 @@ *Bug tracker at https://github.com/giampaolo/psutil/issues* +5.4.7 +===== + +XXXX-XX-XX + +**Bug fixes** + +- 1209_: [OSX] Process.memory_maps() may fail with EINVAL due to poor + task_for_pid() syscall. AccessDenied is now raised instead. + 5.4.6 ===== diff --git a/psutil/_psosx.py b/psutil/_psosx.py index 193f63e00..38f4066e5 100644 --- a/psutil/_psosx.py +++ b/psutil/_psosx.py @@ -334,6 +334,8 @@ def wrapper(self, *args, **kwargs): if err.errno in (errno.EPERM, errno.EACCES): raise AccessDenied(self.pid, self._name) raise + except cext.ZombieProcessError: + raise ZombieProcess(self.pid, self._name, self._ppid) return wrapper @@ -557,8 +559,7 @@ def status(self): @wrap_exceptions def threads(self): - with catch_zombie(self): - rawlist = cext.proc_threads(self.pid) + rawlist = cext.proc_threads(self.pid) retlist = [] for thread_id, utime, stime in rawlist: ntuple = _common.pthread(thread_id, utime, stime) @@ -567,5 +568,4 @@ def threads(self): @wrap_exceptions def memory_maps(self): - with catch_zombie(self): - return cext.proc_memory_maps(self.pid) + return cext.proc_memory_maps(self.pid) diff --git a/psutil/_psutil_osx.c b/psutil/_psutil_osx.c index 55dd64ca5..19508723a 100644 --- a/psutil/_psutil_osx.c +++ b/psutil/_psutil_osx.c @@ -48,6 +48,8 @@ #define PSUTIL_TV2DOUBLE(t) ((t).tv_sec + (t).tv_usec / 1000000.0) +static PyObject *ZombieProcessError; + /* * A wrapper around host_statistics() invoked with HOST_VM_INFO. @@ -71,6 +73,59 @@ psutil_sys_vminfo(vm_statistics_data_t *vmstat) { } +/* + * Return 1 if pid refers to a zombie process else 0. + */ +int +psutil_is_zombie(long pid) +{ + struct kinfo_proc kp; + + if (psutil_get_kinfo_proc(pid, &kp) == -1) + return 0; + return (kp.kp_proc.p_stat == SZOMB) ? 1 : 0; +} + + +/* + * A wrapper around task_for_pid() which sucks big time: + * - it's not documented + * - errno is set only sometimes + * - sometimes errno is ENOENT (?!?) + * - for PIDs != getpid() or PIDs which are not members of the procmod + * it requires root + * As such we can only guess what the heck went wrong and fail either + * with NoSuchProcess, ZombieProcessError or giveup with AccessDenied. + * Here's some history: + * https://github.com/giampaolo/psutil/issues/1181 + * https://github.com/giampaolo/psutil/issues/1209 + * https://github.com/giampaolo/psutil/issues/1291#issuecomment-396062519 + */ +int +psutil_task_for_pid(long pid, mach_port_t *task) +{ + // See: https://github.com/giampaolo/psutil/issues/1181 + kern_return_t err = KERN_SUCCESS; + + err = task_for_pid(mach_task_self(), (pid_t)pid, task); + if (err != KERN_SUCCESS) { + if (psutil_pid_exists(pid) == 0) + NoSuchProcess("task_for_pid() failed"); + else if (psutil_is_zombie(pid) == 1) + PyErr_SetString(ZombieProcessError, "task_for_pid() failed"); + else { + psutil_debug( + "task_for_pid() failed (pid=%ld, err=%i, errno=%i, msg='%s'); " + "setting AccessDenied()", + pid, err, errno, mach_error_string(err)); + AccessDenied("task_for_pid() failed"); + } + return 1; + } + return 0; +} + + /* * Return a Python list of all the PIDs running on the system. */ @@ -336,20 +391,8 @@ psutil_proc_memory_maps(PyObject *self, PyObject *args) { if (! PyArg_ParseTuple(args, "l", &pid)) goto error; - err = task_for_pid(mach_task_self(), (pid_t)pid, &task); - if (err != KERN_SUCCESS) { - if ((err == 5) && (errno == ENOENT)) { - // See: https://github.com/giampaolo/psutil/issues/1181 - psutil_debug("task_for_pid(MACH_PORT_NULL) failed; err=%i, " - "errno=%i, msg='%s'\n", err, errno, - mach_error_string(err)); - AccessDenied(""); - } - else { - psutil_raise_for_pid(pid, "task_for_pid(MACH_PORT_NULL)"); - } + if (psutil_task_for_pid(pid, &task) != 0) goto error; - } while (1) { py_tuple = NULL; @@ -560,7 +603,6 @@ psutil_in_shared_region(mach_vm_address_t addr, cpu_type_t type) { static PyObject * psutil_proc_memory_uss(PyObject *self, PyObject *args) { long pid; - int err; size_t len; cpu_type_t cpu_type; size_t private_pages = 0; @@ -576,14 +618,8 @@ psutil_proc_memory_uss(PyObject *self, PyObject *args) { if (! PyArg_ParseTuple(args, "l", &pid)) return NULL; - err = task_for_pid(mach_task_self(), (pid_t)pid, &task); - if (err != KERN_SUCCESS) { - if (psutil_pid_exists(pid) == 0) - NoSuchProcess(""); - else - AccessDenied(""); + if (psutil_task_for_pid(pid, &task) != 0) return NULL; - } len = sizeof(cpu_type); if (sysctlbyname("sysctl.proc_cputype", &cpu_type, &len, NULL, 0) != 0) @@ -1018,19 +1054,11 @@ psutil_proc_threads(PyObject *self, PyObject *args) { if (py_retlist == NULL) return NULL; - // the argument passed should be a process id if (! PyArg_ParseTuple(args, "l", &pid)) goto error; - // task_for_pid() requires root privileges - err = task_for_pid(mach_task_self(), (pid_t)pid, &task); - if (err != KERN_SUCCESS) { - if (psutil_pid_exists(pid) == 0) - NoSuchProcess(""); - else - AccessDenied(""); + if (psutil_task_for_pid(pid, &task) != 0) goto error; - } info_count = TASK_BASIC_INFO_COUNT; err = task_info(task, TASK_BASIC_INFO, (task_info_t)&tasks_info, @@ -2036,6 +2064,12 @@ init_psutil_osx(void) PyModule_AddIntConstant(module, "TCPS_TIME_WAIT", TCPS_TIME_WAIT); PyModule_AddIntConstant(module, "PSUTIL_CONN_NONE", PSUTIL_CONN_NONE); + // Exception. + ZombieProcessError = PyErr_NewException( + "_psutil_osx.ZombieProcessError", NULL, NULL); + Py_INCREF(ZombieProcessError); + PyModule_AddObject(module, "ZombieProcessError", ZombieProcessError); + psutil_setup(); if (module == NULL) diff --git a/psutil/tests/__main__.py b/psutil/tests/__main__.py index 62fe07428..36554a126 100755 --- a/psutil/tests/__main__.py +++ b/psutil/tests/__main__.py @@ -13,7 +13,6 @@ import contextlib import optparse import os -import ssl import sys import tempfile try: @@ -38,6 +37,7 @@ def install_pip(): try: import pip # NOQA except ImportError: + import ssl f = tempfile.NamedTemporaryFile(suffix='.py') with contextlib.closing(f): print("downloading %s to %s" % (GET_PIP_URL, f.name)) From fdbc40224b3ce113fcc264a1d3180ab0b62cb736 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 15 Jun 2018 00:54:21 +0200 Subject: [PATCH 080/182] OSX / pids(): append() pid 0 in pos 0 --- psutil/_psosx.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/psutil/_psosx.py b/psutil/_psosx.py index 38f4066e5..7cbf40328 100644 --- a/psutil/_psosx.py +++ b/psutil/_psosx.py @@ -309,11 +309,11 @@ def pids(): # https://travis-ci.org/giampaolo/psutil/jobs/309619941 try: Process(0).create_time() - ls.append(0) + ls.insert(0, 0) except NoSuchProcess: pass except AccessDenied: - ls.append(0) + ls.insert(0, 0) return ls From 3612b854956742790f58dcb07a6d50cc4f9b81b8 Mon Sep 17 00:00:00 2001 From: Nikhil Marathe Date: Fri, 15 Jun 2018 21:49:30 +0530 Subject: [PATCH 081/182] Add seconds to thread run times on MacOS. (#1292) Add seconds to thread run times on MacOS (fix #1278) --- psutil/_psutil_osx.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/psutil/_psutil_osx.c b/psutil/_psutil_osx.c index 19508723a..b8e3b85d2 100644 --- a/psutil/_psutil_osx.c +++ b/psutil/_psutil_osx.c @@ -1097,8 +1097,10 @@ psutil_proc_threads(PyObject *self, PyObject *args) { py_tuple = Py_BuildValue( "Iff", j + 1, - (float)basic_info_th->user_time.microseconds / 1000000.0, - (float)basic_info_th->system_time.microseconds / 1000000.0 + basic_info_th->user_time.seconds + \ + (float)basic_info_th->user_time.microseconds / 1000000.0, + basic_info_th->system_time.seconds + \ + (float)basic_info_th->system_time.microseconds / 1000000.0 ); if (!py_tuple) goto error; From d20b9939b985d871177bee2c1271a04a1810a100 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 15 Jun 2018 18:21:17 +0200 Subject: [PATCH 082/182] give CREDITs to @nikhilm for #1278 --- CREDITS | 4 ++++ HISTORY.rst | 2 ++ 2 files changed, 6 insertions(+) diff --git a/CREDITS b/CREDITS index f5f1aa864..a797ff2f0 100644 --- a/CREDITS +++ b/CREDITS @@ -539,3 +539,7 @@ I: 1260 N: Jean-Luc Migot W: https://github.com/jmigot-tehtris I: 1258, 1289 + +N: Nikhil Marathe +W: https://github.com/nikhilm +I: 1278 diff --git a/HISTORY.rst b/HISTORY.rst index 5854fa9fd..ba2f1ed1a 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -9,6 +9,8 @@ XXXX-XX-XX - 1209_: [OSX] Process.memory_maps() may fail with EINVAL due to poor task_for_pid() syscall. AccessDenied is now raised instead. +- 1278_: [OSX] Process.threads() incorrectly return microseconds instead of + seconds. (patch by Nikhil Marathe) 5.4.6 ===== From 196802dd34b7134c1fff99ad4755d0bc0e04cc98 Mon Sep 17 00:00:00 2001 From: Alex Manuskin Date: Tue, 26 Jun 2018 16:19:54 +0300 Subject: [PATCH 083/182] Osx temps (#1284) OSX: add temperatures() and fans() --- psutil/__init__.py | 4 +- psutil/_common.py | 2 +- psutil/_psosx.py | 146 ++++++++++++++++++++ psutil/_psutil_osx.c | 68 ++++++++++ psutil/arch/osx/smc.c | 236 +++++++++++++++++++++++++++++++++ psutil/arch/osx/smc.h | 79 +++++++++++ psutil/tests/test_contracts.py | 4 +- setup.py | 1 + 8 files changed, 535 insertions(+), 5 deletions(-) create mode 100644 psutil/arch/osx/smc.c create mode 100644 psutil/arch/osx/smc.h diff --git a/psutil/__init__.py b/psutil/__init__.py index 5e9a7fb6c..f38592792 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -2175,7 +2175,7 @@ def net_if_stats(): # ===================================================================== -# Linux +# Linux, OSX if hasattr(_psplatform, "sensors_temperatures"): def sensors_temperatures(fahrenheit=False): @@ -2213,7 +2213,7 @@ def convert(n): __all__.append("sensors_temperatures") -# Linux +# Linux, OSX if hasattr(_psplatform, "sensors_fans"): def sensors_fans(): diff --git a/psutil/_common.py b/psutil/_common.py index e65d39e3b..b72139396 100644 --- a/psutil/_common.py +++ b/psutil/_common.py @@ -196,7 +196,7 @@ class BatteryTime(enum.IntEnum): 'shwtemp', ['label', 'current', 'high', 'critical']) # psutil.sensors_battery() sbattery = namedtuple('sbattery', ['percent', 'secsleft', 'power_plugged']) -# psutil.sensors_battery() +# psutil.sensors_fans() sfan = namedtuple('sfan', ['label', 'current']) # --- for Process methods diff --git a/psutil/_psosx.py b/psutil/_psosx.py index 7cbf40328..90c8be438 100644 --- a/psutil/_psosx.py +++ b/psutil/_psosx.py @@ -6,6 +6,7 @@ import contextlib import errno +import collections import functools import os from socket import AF_INET @@ -62,6 +63,110 @@ cext.SZOMB: _common.STATUS_ZOMBIE, } +temperatures = ( + # group, key, label + + # --- CPU + ("CPU", "TCXC", "PECI CPU"), + ("CPU", "TCXc", "PECI CPU"), + ("CPU", "TC0P", "CPU 1 Proximity"), + ("CPU", "TC0H", "CPU 1 Heatsink"), + ("CPU", "TC0D", "CPU 1 Package"), + ("CPU", "TC0E", "CPU 1"), + ("CPU", "TC1C", "CPU Core 1"), + ("CPU", "TC2C", "CPU Core 2"), + ("CPU", "TC3C", "CPU Core 3"), + ("CPU", "TC4C", "CPU Core 4"), + ("CPU", "TC5C", "CPU Core 5"), + ("CPU", "TC6C", "CPU Core 6"), + ("CPU", "TC7C", "CPU Core 7"), + ("CPU", "TC8C", "CPU Core 8"), + ("CPU", "TCAH", "CPU 1 Heatsink Alt."), + ("CPU", "TCAD", "CPU 1 Package Alt."), + ("CPU", "TC1P", "CPU 2 Proximity"), + ("CPU", "TC1H", "CPU 2 Heatsink"), + ("CPU", "TC1D", "CPU 2 Package"), + ("CPU", "TC1E", "CPU 2"), + ("CPU", "TCBH", "CPU 2 Heatsink Alt."), + ("CPU", "TCBD", "CPU 2 Package Alt."), + + ("CPU", "TCSC", "PECI SA"), + ("CPU", "TCSc", "PECI SA"), + ("CPU", "TCSA", "PECI SA"), + + # --- GPU + ("GPU", "TCGC", "PECI GPU"), + ("GPU", "TCGc", "PECI GPU"), + ("GPU", "TG0P", "GPU Proximity"), + ("GPU", "TG0D", "GPU Die"), + ("GPU", "TG1D", "GPU Die"), + ("GPU", "TG0H", "GPU Heatsink"), + ("GPU", "TG1H", "GPU Heatsink"), + + # --- Memory + ("Memory", "Ts0S", "Memory Proximity"), + ("Memory", "TM0P", "Mem Bank A1"), + ("Memory", "TM1P", "Mem Bank A2"), + ("Memory", "TM8P", "Mem Bank B1"), + ("Memory", "TM9P", "Mem Bank B2"), + ("Memory", "TM0S", "Mem Module A1"), + ("Memory", "TM1S", "Mem Module A2"), + ("Memory", "TM8S", "Mem Module B1"), + ("Memory", "TM9S", "Mem Module B2"), + + # --- HDD + ("HDD", "TH0P", "HDD Bay 1"), + ("HDD", "TH1P", "HDD Bay 2"), + ("HDD", "TH2P", "HDD Bay 3"), + ("HDD", "TH3P", "HDD Bay 4"), + + # --- Battery + ("Battery", "TB0T", "Battery TS_MAX"), + ("Battery", "TB1T", "Battery 1"), + ("Battery", "TB2T", "Battery 2"), + ("Battery", "TB3T", "Battery"), + + # --- Others + ("Others", "TN0D", "Northbridge Die"), + ("Others", "TN0P", "Northbridge Proximity 1"), + ("Others", "TN1P", "Northbridge Proximity 2"), + ("Others", "TN0C", "MCH Die"), + ("Others", "TN0H", "MCH Heatsink"), + ("Others", "TP0D", "PCH Die"), + ("Others", "TPCD", "PCH Die"), + ("Others", "TP0P", "PCH Proximity"), + + ("Others", "TA0P", "Airflow 1"), + ("Others", "TA1P", "Airflow 2"), + ("Others", "Th0H", "Heatpipe 1"), + ("Others", "Th1H", "Heatpipe 2"), + ("Others", "Th2H", "Heatpipe 3"), + + ("Others", "Tm0P", "Mainboard Proximity"), + ("Others", "Tp0P", "Powerboard Proximity"), + ("Others", "Ts0P", "Palm Rest"), + ("Others", "Tb0P", "BLC Proximity"), + + ("Others", "TL0P", "LCD Proximity"), + ("Others", "TW0P", "Airport Proximity"), + ("Others", "TO0P", "Optical Drive"), + + ("Others", "Tp0P", "Power Supply 1"), + ("Others", "Tp0C", "Power Supply 1 Alt."), + ("Others", "Tp1P", "Power Supply 2"), + ("Others", "Tp1C", "Power Supply 2 Alt."), + ("Others", "Tp2P", "Power Supply 3"), + ("Others", "Tp3P", "Power Supply 4"), + ("Others", "Tp4P", "Power Supply 5"), + ("Others", "Tp5P", "Power Supply 6"), + + ("Others", "TS0C", "Expansion Slots"), + ("Others", "TA0S", "PCI Slot 1 Pos 1"), + ("Others", "TA1S", "PCI Slot 1 Pos 2"), + ("Others", "TA2S", "PCI Slot 2 Pos 1"), + ("Others", "TA3S", "PCI Slot 2 Pos 2"), +) + kinfo_proc_map = dict( ppid=0, ruid=1, @@ -212,6 +317,35 @@ def disk_partitions(all=False): # ===================================================================== +def sensors_temperatures(): + """Returns a dictionary of regions of temperature sensors: + CPU/GPU/Memory/Others + Each entry contains a list of results of all the successfully polled + SMC keys from the system. + + References for SMC keys and meaning: + + https://stackoverflow.com/questions/28568775/ + description-for-apples-smc-keys/31033665#31033665 + + https://github.com/Chris911/iStats/blob/ + 09b159f85a9481b59f347a37259f6d272f65cc05/lib/iStats/smc.rb + + http://web.archive.org/web/20140714090133/http://www.parhelia.ch:80/ + blog/statics/k3_keys.html + """ + ret = collections.defaultdict(list) + + for group, key, label in temperatures: + result = cext.smc_get_temperature(key) + result = round(result, 1) + if result <= 0: + continue + ret[group].append((label, result, None, None)) + + return dict(ret) + + def sensors_battery(): """Return battery information. """ @@ -230,6 +364,18 @@ def sensors_battery(): return _common.sbattery(percent, secsleft, power_plugged) +def sensors_fans(): + """Return fans speed information. + """ + ret = collections.defaultdict(list) + rawlist = cext.sensors_fans() + if rawlist is not None: + for fan in rawlist: + ret["Fans"].append(_common.sfan(fan[0], fan[1])) + + return dict(ret) + + # ===================================================================== # --- network # ===================================================================== diff --git a/psutil/_psutil_osx.c b/psutil/_psutil_osx.c index b8e3b85d2..584df3b83 100644 --- a/psutil/_psutil_osx.c +++ b/psutil/_psutil_osx.c @@ -44,6 +44,7 @@ #include "_psutil_common.h" #include "_psutil_posix.h" #include "arch/osx/process_info.h" +#include "arch/osx/smc.h" #define PSUTIL_TV2DOUBLE(t) ((t).tv_sec + (t).tv_usec / 1000000.0) @@ -896,6 +897,68 @@ psutil_boot_time(PyObject *self, PyObject *args) { return Py_BuildValue("f", (float)boot_time); } +/* + * Return a Python float indicating the value of the temperature + * measured by an SMC key + */ +static PyObject * +psutil_smc_get_temperature(PyObject *self, PyObject *args) { + char* key; + float temp; + + if (! PyArg_ParseTuple(args, "s", &key)) { + return NULL; + } + temp = SMCGetTemperature(key); + return Py_BuildValue("d", temp); +} + + +/* + * Return a Python list of tuples of fan label and speed + */ +static PyObject * +psutil_sensors_fans(PyObject *self, PyObject *args) { + int key; + int speed; + char fan[7]; + int fan_count; + PyObject *py_tuple = NULL; + PyObject *py_retlist = PyList_New(0); + + if (py_retlist == NULL) + return NULL; + + fan_count = SMCGetFanNumber(SMC_KEY_FAN_NUM); + if (fan_count < 0) { + fan_count = 0; + } + for (key =0; key < fan_count; key++) { + sprintf(fan, "Fan %d", key); + speed = SMCGetFanSpeed(key); + if (speed < 0) { + continue; + } + py_tuple = Py_BuildValue( + "(si)", + fan, // label + speed // value + ); + if (!py_tuple) + goto error; + + if (PyList_Append(py_retlist, py_tuple)) { + goto error; + } + Py_XDECREF(py_tuple); + } + + return py_retlist; +error: + Py_XDECREF(py_tuple); + Py_XDECREF(py_retlist); + return NULL; +} /* * Return a list of tuples including device, mount point and fs type @@ -1828,6 +1891,7 @@ psutil_cpu_stats(PyObject *self, PyObject *args) { } + /* * Return battery information. */ @@ -1980,6 +2044,10 @@ PsutilMethods[] = { "Return currently connected users as a list of tuples"}, {"cpu_stats", psutil_cpu_stats, METH_VARARGS, "Return CPU statistics"}, + {"smc_get_temperature", psutil_smc_get_temperature, METH_VARARGS, + "Temperature of SMC key as float"}, + {"sensors_fans", psutil_sensors_fans, METH_VARARGS, + "Return the RPM of the fan with SMC key"}, {"sensors_battery", psutil_sensors_battery, METH_VARARGS, "Return battery information."}, diff --git a/psutil/arch/osx/smc.c b/psutil/arch/osx/smc.c new file mode 100644 index 000000000..fb5668a1e --- /dev/null +++ b/psutil/arch/osx/smc.c @@ -0,0 +1,236 @@ +/* + * Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + * + * Interface to SMC API, needed in order to collect sensors stats. + */ + +#include +#include +#include + +#include "smc.h" + +UInt32 _strtoul(char *str, int size, int base) +{ + UInt32 total = 0; + int i; + + for (i = 0; i < size; i++) { + if (base == 16) + total += str[i] << (size - 1 - i) * 8; + else + total += (unsigned char) (str[i] << (size - 1 - i) * 8); + } + return total; +} + + +float _strtof(unsigned char *str, int size, int e) +{ + float total = 0; + int i; + + for (i = 0; i < size; i++) { + if (i == (size - 1)) + total += (str[i] & 0xff) >> e; + else + total += str[i] << (size - 1 - i) * (8 - e); + } + + total += (str[size-1] & 0x03) * 0.25; + + return total; +} + + +void _ultostr(char *str, UInt32 val) +{ + str[0] = '\0'; + sprintf(str, "%c%c%c%c", + (unsigned int) val >> 24, + (unsigned int) val >> 16, + (unsigned int) val >> 8, + (unsigned int) val); +} + + +kern_return_t SMCOpen(io_connect_t * pconn) +{ + kern_return_t result; + io_iterator_t iterator; + io_object_t device; + + CFMutableDictionaryRef matchingDictionary = IOServiceMatching("AppleSMC"); + result = IOServiceGetMatchingServices( + kIOMasterPortDefault, + matchingDictionary, + &iterator + ); + if (result != kIOReturnSuccess) { + return 1; + } + + device = IOIteratorNext(iterator); + IOObjectRelease(iterator); + if (device == 0) { + return 1; + } + + result = IOServiceOpen(device, mach_task_self(), 0, pconn); + IOObjectRelease(device); + if (result != kIOReturnSuccess) { + return 1; + } + + return kIOReturnSuccess; +} + + +kern_return_t SMCClose(io_connect_t conn) +{ + return IOServiceClose(conn); +} + + +kern_return_t SMCCall(io_connect_t conn, + int index, SMCKeyData_t *inputStructure, SMCKeyData_t *outputStructure) +{ + size_t structureInputSize; + size_t structureOutputSize; + + structureInputSize = sizeof(SMCKeyData_t); + structureOutputSize = sizeof(SMCKeyData_t); + +#if MAC_OS_X_VERSION_10_5 + return IOConnectCallStructMethod( + conn, index, + inputStructure, + structureInputSize, + outputStructure, + &structureOutputSize); +#else + return IOConnectMethodStructureIStructureO( + conn, index, + structureInputSize, + &structureOutputSize, + inputStructure, + outputStructure); +#endif +} + + +kern_return_t SMCReadKey(io_connect_t conn, UInt32Char_t key, SMCVal_t *val) { + kern_return_t result; + SMCKeyData_t inputStructure; + SMCKeyData_t outputStructure; + + memset(&inputStructure, 0, sizeof(SMCKeyData_t)); + memset(&outputStructure, 0, sizeof(SMCKeyData_t)); + memset(val, 0, sizeof(SMCVal_t)); + + inputStructure.key = _strtoul(key, 4, 16); + inputStructure.data8 = SMC_CMD_READ_KEYINFO; + + result = SMCCall( + conn, + KERNEL_INDEX_SMC, + &inputStructure, + &outputStructure + ); + if (result != kIOReturnSuccess) + return result; + + val->dataSize = outputStructure.keyInfo.dataSize; + _ultostr(val->dataType, outputStructure.keyInfo.dataType); + inputStructure.keyInfo.dataSize = val->dataSize; + inputStructure.data8 = SMC_CMD_READ_BYTES; + + result = SMCCall( + conn, + KERNEL_INDEX_SMC, + &inputStructure, + &outputStructure + ); + if (result != kIOReturnSuccess) + return result; + + memcpy(val->bytes, outputStructure.bytes, sizeof(outputStructure.bytes)); + + return kIOReturnSuccess; +} + + +double SMCGetTemperature(char *key) { + io_connect_t conn; + SMCVal_t val; + kern_return_t result; + int intValue; + + result = SMCOpen(&conn); + if (result != kIOReturnSuccess) { + return 0.0; + } + + result = SMCReadKey(conn, key, &val); + if (result == kIOReturnSuccess) { + // read succeeded - check returned value + if (val.dataSize > 0) { + if (strcmp(val.dataType, DATATYPE_SP78) == 0) { + // convert sp78 value to temperature + intValue = val.bytes[0] * 256 + (unsigned char)val.bytes[1]; + SMCClose(conn); + return intValue / 256.0; + } + } + } + // read failed + SMCClose(conn); + return 0.0; +} + + +float SMCGetFanSpeed(int fanNum) +{ + SMCVal_t val; + kern_return_t result; + UInt32Char_t key; + io_connect_t conn; + + result = SMCOpen(&conn); + if (result != kIOReturnSuccess) { + return -1; + } + + sprintf(key, SMC_KEY_FAN_SPEED, fanNum); + result = SMCReadKey(conn, key, &val); + if (result != kIOReturnSuccess) { + SMCClose(conn); + return -1; + } + SMCClose(conn); + return _strtof((unsigned char *)val.bytes, val.dataSize, 2); +} + + + +int SMCGetFanNumber(char *key) +{ + SMCVal_t val; + kern_return_t result; + io_connect_t conn; + + result = SMCOpen(&conn); + if (result != kIOReturnSuccess) { + return 0; + } + + result = SMCReadKey(conn, key, &val); + if (result != kIOReturnSuccess) { + SMCClose(conn); + return 0; + } + SMCClose(conn); + return _strtoul((char *)val.bytes, val.dataSize, 10); +} diff --git a/psutil/arch/osx/smc.h b/psutil/arch/osx/smc.h new file mode 100644 index 000000000..acd399c2e --- /dev/null +++ b/psutil/arch/osx/smc.h @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + */ + +#ifndef __SMC_H__ +#define __SMC_H__ + +#define KERNEL_INDEX_SMC 2 + +#define SMC_CMD_READ_BYTES 5 +#define SMC_CMD_WRITE_BYTES 6 +#define SMC_CMD_READ_INDEX 8 +#define SMC_CMD_READ_KEYINFO 9 +#define SMC_CMD_READ_PLIMIT 11 +#define SMC_CMD_READ_VERS 12 + + +#define DATATYPE_FPE2 "fpe2" +#define DATATYPE_UINT8 "ui8 " +#define DATATYPE_UINT16 "ui16" +#define DATATYPE_UINT32 "ui32" +#define DATATYPE_SP78 "sp78" + +// Fans SMC key values +#define SMC_KEY_FAN_SPEED "F%dAc" +#define SMC_KEY_FAN_NUM "FNum" + +typedef struct { + char major; + char minor; + char build; + char reserved[1]; + UInt16 release; +} SMCKeyData_vers_t; + +typedef struct { + UInt16 version; + UInt16 length; + UInt32 cpuPLimit; + UInt32 gpuPLimit; + UInt32 memPLimit; +} SMCKeyData_pLimitData_t; + +typedef struct { + UInt32 dataSize; + UInt32 dataType; + char dataAttributes; +} SMCKeyData_keyInfo_t; + +typedef char SMCBytes_t[32]; + +typedef struct { + UInt32 key; + SMCKeyData_vers_t vers; + SMCKeyData_pLimitData_t pLimitData; + SMCKeyData_keyInfo_t keyInfo; + char result; + char status; + char data8; + UInt32 data32; + SMCBytes_t bytes; +} SMCKeyData_t; + +typedef char UInt32Char_t[5]; + +typedef struct { + UInt32Char_t key; + UInt32 dataSize; + UInt32Char_t dataType; + SMCBytes_t bytes; +} SMCVal_t; + +double SMCGetTemperature(char *key); +int SMCGetFanNumber(char *key); +float SMCGetFanSpeed(int fanNum); + +#endif diff --git a/psutil/tests/test_contracts.py b/psutil/tests/test_contracts.py index 0dfb3e2a8..912e811f3 100755 --- a/psutil/tests/test_contracts.py +++ b/psutil/tests/test_contracts.py @@ -116,10 +116,10 @@ def test_cpu_freq(self): self.assertEqual(hasattr(psutil, "cpu_freq"), linux or OSX or WINDOWS) def test_sensors_temperatures(self): - self.assertEqual(hasattr(psutil, "sensors_temperatures"), LINUX) + self.assertEqual(hasattr(psutil, "sensors_temperatures"), LINUX or OSX) def test_sensors_fans(self): - self.assertEqual(hasattr(psutil, "sensors_fans"), LINUX) + self.assertEqual(hasattr(psutil, "sensors_fans"), LINUX or OSX) def test_battery(self): self.assertEqual(hasattr(psutil, "sensors_battery"), diff --git a/setup.py b/setup.py index fffefd2f7..b0343212b 100755 --- a/setup.py +++ b/setup.py @@ -152,6 +152,7 @@ def get_winver(): sources=sources + [ 'psutil/_psutil_osx.c', 'psutil/arch/osx/process_info.c', + 'psutil/arch/osx/smc.c', ], define_macros=macros, extra_link_args=[ From 00c12b954749ce6d771f69ddf29cf4cb87f290ca Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 26 Jun 2018 15:20:38 +0200 Subject: [PATCH 084/182] little refactoring --- psutil/_common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/psutil/_common.py b/psutil/_common.py index e65d39e3b..fdc94b153 100644 --- a/psutil/_common.py +++ b/psutil/_common.py @@ -80,7 +80,7 @@ OPENBSD = sys.platform.startswith("openbsd") NETBSD = sys.platform.startswith("netbsd") BSD = FREEBSD or OPENBSD or NETBSD -SUNOS = sys.platform.startswith("sunos") or sys.platform.startswith("solaris") +SUNOS = sys.platform.startswith(("sunos", "solaris")) AIX = sys.platform.startswith("aix") From 93b1da90ee48630d57102faa989941ce46636644 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 26 Jun 2018 15:35:46 +0200 Subject: [PATCH 085/182] #1284: give credits to @amanusk + some minor adjustments --- CREDITS | 4 ++++ HISTORY.rst | 7 ++++++- docs/index.rst | 8 ++++++-- psutil/_psosx.py | 34 ++++++++++++++-------------------- psutil/_psutil_osx.c | 22 ++++++++-------------- psutil/arch/osx/smc.c | 6 ++++-- 6 files changed, 42 insertions(+), 39 deletions(-) diff --git a/CREDITS b/CREDITS index a797ff2f0..fd4f53d64 100644 --- a/CREDITS +++ b/CREDITS @@ -543,3 +543,7 @@ I: 1258, 1289 N: Nikhil Marathe W: https://github.com/nikhilm I: 1278 + +N: Alex Manuskin +W: https://github.com/amanusk +I: 1284 diff --git a/HISTORY.rst b/HISTORY.rst index ba2f1ed1a..3ba713145 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,10 +1,15 @@ *Bug tracker at https://github.com/giampaolo/psutil/issues* -5.4.7 +5.5.0 ===== XXXX-XX-XX +**Enhancements** + +- 1284_: [OSX] added support for sensors_temperatures() and sensors_fans(). + (patch by Alex Manuskin) + **Bug fixes** - 1209_: [OSX] Process.memory_maps() may fail with EINVAL due to poor diff --git a/docs/index.rst b/docs/index.rst index bde3d453c..208f59e9c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -698,10 +698,12 @@ Sensors See also `temperatures.py `__ and `sensors.py `__ for an example application. - Availability: Linux + Availability: Linux, OSX .. versionadded:: 5.1.0 + .. versionchanged:: 5.5.0: added OSX support + .. warning:: this API is experimental. Backward incompatible changes may occur if @@ -722,10 +724,12 @@ Sensors See also `fans.py `__ and `sensors.py `__ for an example application. - Availability: Linux + Availability: Linux, OSX .. versionadded:: 5.2.0 + .. versionchanged:: 5.5.0: added OSX support + .. warning:: this API is experimental. Backward incompatible changes may occur if diff --git a/psutil/_psosx.py b/psutil/_psosx.py index 90c8be438..a0101df07 100644 --- a/psutil/_psosx.py +++ b/psutil/_psosx.py @@ -4,9 +4,9 @@ """OSX platform implementation.""" +import collections import contextlib import errno -import collections import functools import os from socket import AF_INET @@ -63,9 +63,7 @@ cext.SZOMB: _common.STATUS_ZOMBIE, } -temperatures = ( - # group, key, label - +SMC_TEMPERATURES = ( # --- CPU ("CPU", "TCXC", "PECI CPU"), ("CPU", "TCXc", "PECI CPU"), @@ -319,36 +317,34 @@ def disk_partitions(all=False): def sensors_temperatures(): """Returns a dictionary of regions of temperature sensors: - CPU/GPU/Memory/Others - Each entry contains a list of results of all the successfully polled - SMC keys from the system. - + CPU/GPU/Memory/HDD/Battery/Others. + Each entry contains a list of results of all the SMC keys successfully + polled from the system. References for SMC keys and meaning: - https://stackoverflow.com/questions/28568775/ + * https://stackoverflow.com/questions/28568775/ description-for-apples-smc-keys/31033665#31033665 - https://github.com/Chris911/iStats/blob/ + * https://github.com/Chris911/iStats/blob/ 09b159f85a9481b59f347a37259f6d272f65cc05/lib/iStats/smc.rb - http://web.archive.org/web/20140714090133/http://www.parhelia.ch:80/ + * http://web.archive.org/web/20140714090133/http://www.parhelia.ch:80/ blog/statics/k3_keys.html """ + # TODO: this should be rewritten in C: + # https://github.com/giampaolo/psutil/pull/1284#issuecomment-399480983 ret = collections.defaultdict(list) - - for group, key, label in temperatures: + for group, key, label in SMC_TEMPERATURES: result = cext.smc_get_temperature(key) result = round(result, 1) if result <= 0: continue ret[group].append((label, result, None, None)) - return dict(ret) def sensors_battery(): - """Return battery information. - """ + """Return battery information.""" try: percent, minsleft, power_plugged = cext.sensors_battery() except NotImplementedError: @@ -369,10 +365,8 @@ def sensors_fans(): """ ret = collections.defaultdict(list) rawlist = cext.sensors_fans() - if rawlist is not None: - for fan in rawlist: - ret["Fans"].append(_common.sfan(fan[0], fan[1])) - + for fan in rawlist: + ret["Fans"].append(_common.sfan(fan[0], fan[1])) return dict(ret) diff --git a/psutil/_psutil_osx.c b/psutil/_psutil_osx.c index 584df3b83..15f4d4f69 100644 --- a/psutil/_psutil_osx.c +++ b/psutil/_psutil_osx.c @@ -930,33 +930,27 @@ psutil_sensors_fans(PyObject *self, PyObject *args) { return NULL; fan_count = SMCGetFanNumber(SMC_KEY_FAN_NUM); - if (fan_count < 0) { + if (fan_count < 0) fan_count = 0; - } - for (key =0; key < fan_count; key++) { + + for (key = 0; key < fan_count; key++) { sprintf(fan, "Fan %d", key); speed = SMCGetFanSpeed(key); - if (speed < 0) { + if (speed < 0) continue; - } - py_tuple = Py_BuildValue( - "(si)", - fan, // label - speed // value - ); + py_tuple = Py_BuildValue("(si)", fan, speed); if (!py_tuple) goto error; - - if (PyList_Append(py_retlist, py_tuple)) { + if (PyList_Append(py_retlist, py_tuple)) goto error; - } Py_XDECREF(py_tuple); } return py_retlist; + error: Py_XDECREF(py_tuple); - Py_XDECREF(py_retlist); + Py_DECREF(py_retlist); return NULL; } diff --git a/psutil/arch/osx/smc.c b/psutil/arch/osx/smc.c index fb5668a1e..00de0a372 100644 --- a/psutil/arch/osx/smc.c +++ b/psutil/arch/osx/smc.c @@ -105,14 +105,16 @@ kern_return_t SMCCall(io_connect_t conn, #if MAC_OS_X_VERSION_10_5 return IOConnectCallStructMethod( - conn, index, + conn, + index, inputStructure, structureInputSize, outputStructure, &structureOutputSize); #else return IOConnectMethodStructureIStructureO( - conn, index, + conn, + index, structureInputSize, &structureOutputSize, inputStructure, From 44ff3cda273d31abbdd36d51c20efaee61da0622 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 26 Jun 2018 23:53:16 +0200 Subject: [PATCH 086/182] appveyor: remove py 3.4 and add 3.7 --- appveyor.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 436faadbc..dbf19ae44 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -20,10 +20,6 @@ environment: PYTHON_VERSION: "2.7.x" PYTHON_ARCH: "32" - - PYTHON: "C:\\Python34" - PYTHON_VERSION: "3.4.x" - PYTHON_ARCH: "32" - - PYTHON: "C:\\Python35" PYTHON_VERSION: "3.5.x" PYTHON_ARCH: "32" @@ -32,16 +28,16 @@ environment: PYTHON_VERSION: "3.6.x" PYTHON_ARCH: "32" + - PYTHON: "C:\\Python37" + PYTHON_VERSION: "3.7.x" + PYTHON_ARCH: "32" + # 64 bits - PYTHON: "C:\\Python27-x64" PYTHON_VERSION: "2.7.x" PYTHON_ARCH: "64" - - PYTHON: "C:\\Python34-x64" - PYTHON_VERSION: "3.4.x" - PYTHON_ARCH: "64" - - PYTHON: "C:\\Python35-x64" PYTHON_VERSION: "3.5.x" PYTHON_ARCH: "64" @@ -50,6 +46,10 @@ environment: PYTHON_VERSION: "3.6.x" PYTHON_ARCH: "64" + - PYTHON: "C:\\Python37-x64" + PYTHON_VERSION: "3.7.x" + PYTHON_ARCH: "64" + # Also build on a Python version not pre-installed by Appveyor. # See: https://github.com/ogrisel/python-appveyor-demo/issues/10 # - PYTHON: "C:\\Python266" From f6def245a832feb6b15f53dfc006ba40cd38c98a Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 27 Jun 2018 00:00:46 +0200 Subject: [PATCH 087/182] try to fix occasional children() failure on Win: https://ci.appveyor.com/project/giampaolo/psutil/build/job/je3qyldbb86ff66h --- psutil/tests/test_process.py | 1 + 1 file changed, 1 insertion(+) diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index 6f58be6d2..e295c95b1 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -1059,6 +1059,7 @@ def test_parent_disappeared(self): self.assertIsNone(p.parent()) def test_children(self): + reap_children(recursive=True) p = psutil.Process() self.assertEqual(p.children(), []) self.assertEqual(p.children(recursive=True), []) From 7a3ed446447ec845a7d03b32233b4ed7b48f3c26 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 27 Jun 2018 00:01:18 +0200 Subject: [PATCH 088/182] apveyor: reset py 3.4 and remove 3.7 (not available yet) --- appveyor.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index dbf19ae44..436faadbc 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -20,6 +20,10 @@ environment: PYTHON_VERSION: "2.7.x" PYTHON_ARCH: "32" + - PYTHON: "C:\\Python34" + PYTHON_VERSION: "3.4.x" + PYTHON_ARCH: "32" + - PYTHON: "C:\\Python35" PYTHON_VERSION: "3.5.x" PYTHON_ARCH: "32" @@ -28,16 +32,16 @@ environment: PYTHON_VERSION: "3.6.x" PYTHON_ARCH: "32" - - PYTHON: "C:\\Python37" - PYTHON_VERSION: "3.7.x" - PYTHON_ARCH: "32" - # 64 bits - PYTHON: "C:\\Python27-x64" PYTHON_VERSION: "2.7.x" PYTHON_ARCH: "64" + - PYTHON: "C:\\Python34-x64" + PYTHON_VERSION: "3.4.x" + PYTHON_ARCH: "64" + - PYTHON: "C:\\Python35-x64" PYTHON_VERSION: "3.5.x" PYTHON_ARCH: "64" @@ -46,10 +50,6 @@ environment: PYTHON_VERSION: "3.6.x" PYTHON_ARCH: "64" - - PYTHON: "C:\\Python37-x64" - PYTHON_VERSION: "3.7.x" - PYTHON_ARCH: "64" - # Also build on a Python version not pre-installed by Appveyor. # See: https://github.com/ogrisel/python-appveyor-demo/issues/10 # - PYTHON: "C:\\Python266" From 2f60e997fc1c500e0c245979bbecf2761f36e820 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 26 Jun 2018 16:57:36 -0700 Subject: [PATCH 089/182] Rename OSX to macOS (#1298) rename OSX to macOS --- .ci/README | 2 +- .ci/travis/run.sh | 2 +- .coveragerc | 2 +- .travis.yml | 2 +- CREDITS | 4 +- DEVGUIDE.rst | 10 +-- HISTORY.rst | 99 +++++++++++---------- IDEAS | 6 +- INSTALL.rst | 6 +- MANIFEST.in | 12 +-- Makefile | 2 +- README.rst | 6 +- docs/index.rst | 57 ++++++------ psutil/__init__.py | 37 ++++---- psutil/_common.py | 9 +- psutil/_exceptions.py | 2 +- psutil/{_psosx.py => _psmacos.py} | 10 +-- psutil/_psutil_bsd.c | 2 +- psutil/{_psutil_osx.c => _psutil_macos.c} | 32 +++---- psutil/_psutil_posix.c | 20 ++--- psutil/arch/{osx => macos}/process_info.c | 6 +- psutil/arch/{osx => macos}/process_info.h | 0 psutil/arch/{osx => macos}/smc.c | 0 psutil/arch/{osx => macos}/smc.h | 0 psutil/tests/README.rst | 2 +- psutil/tests/__init__.py | 8 +- psutil/tests/test_connections.py | 10 +-- psutil/tests/test_contracts.py | 22 ++--- psutil/tests/{test_osx.py => test_macos.py} | 12 +-- psutil/tests/test_memory_leaks.py | 10 +-- psutil/tests/test_posix.py | 10 +-- psutil/tests/test_process.py | 24 ++--- psutil/tests/test_system.py | 12 +-- psutil/tests/test_unicode.py | 6 +- scripts/internal/bench_oneshot.py | 2 +- scripts/internal/print_announce.py | 6 +- scripts/iotop.py | 2 +- scripts/pmap.py | 2 +- scripts/procsmem.py | 2 +- setup.py | 16 ++-- 40 files changed, 245 insertions(+), 229 deletions(-) rename psutil/{_psosx.py => _psmacos.py} (98%) rename psutil/{_psutil_osx.c => _psutil_macos.c} (98%) rename psutil/arch/{osx => macos}/process_info.c (98%) rename psutil/arch/{osx => macos}/process_info.h (100%) rename psutil/arch/{osx => macos}/smc.c (100%) rename psutil/arch/{osx => macos}/smc.h (100%) rename psutil/tests/{test_osx.py => test_macos.py} (97%) diff --git a/.ci/README b/.ci/README index 86b72afb8..810fe6380 100644 --- a/.ci/README +++ b/.ci/README @@ -1,3 +1,3 @@ This directory contains support scripts for Travis and Appveyor continuous integration services. -Travis is used to run tests on Linux and OSX, Appveyor runs tests on Windows. +Travis is used to run tests on Linux and macOS, Appveyor runs tests on Windows. diff --git a/.ci/travis/run.sh b/.ci/travis/run.sh index 1501387a5..06cc5b7e6 100755 --- a/.ci/travis/run.sh +++ b/.ci/travis/run.sh @@ -5,7 +5,7 @@ set -x PYVER=`python -c 'import sys; print(".".join(map(str, sys.version_info[:2])))'` -# setup OSX +# setup macOS if [[ "$(uname -s)" == 'Darwin' ]]; then if which pyenv > /dev/null; then eval "$(pyenv init -)" diff --git a/.coveragerc b/.coveragerc index 7d3f185f5..c6772e953 100644 --- a/.coveragerc +++ b/.coveragerc @@ -21,7 +21,7 @@ exclude_lines = if LITTLE_ENDIAN: if NETBSD if OPENBSD - if OSX + if MACOS if ppid_map is None: if PY3: if SUNOS diff --git a/.travis.yml b/.travis.yml index 9289eb6b7..86cc73ec0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,7 +9,7 @@ matrix: - python: 3.4 - python: 3.5 - python: 3.6 - # OSX + # macOS - language: generic os: osx env: PYVER=py27 diff --git a/CREDITS b/CREDITS index fd4f53d64..89a91008e 100644 --- a/CREDITS +++ b/CREDITS @@ -34,7 +34,7 @@ Github usernames of people to CC on github when in need of help. - glebius, Gleb Smirnoff (#1013) - sunpoet, Po-Chuan Hsieh (pkg maintainer, #1105) - kostikbel, Konstantin Belousov (#1105) -- OSX: +- macOS: - whitlockjc, Jeremy Whitlock - Windows: - mrjefftang, Jeff Tang @@ -65,7 +65,7 @@ I: 340, 529, 616, 653, 654, 648, 641 N: Jeremy Whitlock E: jcscoobyrs@gmail.com -D: great help with OSX C development. +D: great help with macOS C development. I: 125, 150, 174, 206 N: Landry Breuil diff --git a/DEVGUIDE.rst b/DEVGUIDE.rst index 2d48ced27..f604480e5 100644 --- a/DEVGUIDE.rst +++ b/DEVGUIDE.rst @@ -139,10 +139,10 @@ All of the services listed below are automatically run on ``git push``. Unit tests ---------- -Tests are automatically run for every GIT push on **Linux**, **OSX** and +Tests are automatically run for every GIT push on **Linux**, **macOS** and **Windows** by using: -- `Travis `_ (Linux, OSX) +- `Travis `_ (Linux, macOS) - `Appveyor `_ (Windows) Test files controlling these are @@ -153,15 +153,15 @@ Both services run psutil test suite against all supported python version (2.6 - 3.6). Two icons in the home page (README) always show the build status: -.. image:: https://img.shields.io/travis/giampaolo/psutil/master.svg?maxAge=3600&label=Linux%20/%20OSX +.. image:: https://img.shields.io/travis/giampaolo/psutil/master.svg?maxAge=3600&label=Linux%20/%20macOS :target: https://travis-ci.org/giampaolo/psutil - :alt: Linux tests (Travis) + :alt: Linux and macOS tests (Travis) .. image:: https://img.shields.io/appveyor/ci/giampaolo/psutil/master.svg?maxAge=3600&label=Windows :target: https://ci.appveyor.com/project/giampaolo/psutil :alt: Windows tests (Appveyor) -OSX, BSD, AIX and Solaris are currently tested manually (sigh!). +BSD, AIX and Solaris are currently tested manually. Test coverage ------------- diff --git a/HISTORY.rst b/HISTORY.rst index 3ba713145..67f967eda 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -7,14 +7,16 @@ XXXX-XX-XX **Enhancements** -- 1284_: [OSX] added support for sensors_temperatures() and sensors_fans(). +- 1284_: [macOS] added support for sensors_temperatures() and sensors_fans(). (patch by Alex Manuskin) +- 1286_: [macOS] psutil.OSX constant is now deprecated in favor of new + psutil.MACOS. **Bug fixes** -- 1209_: [OSX] Process.memory_maps() may fail with EINVAL due to poor +- 1209_: [macOS] Process.memory_maps() may fail with EINVAL due to poor task_for_pid() syscall. AccessDenied is now raised instead. -- 1278_: [OSX] Process.threads() incorrectly return microseconds instead of +- 1278_: [macOS] Process.threads() incorrectly return microseconds instead of seconds. (patch by Nikhil Marathe) 5.4.6 @@ -101,7 +103,7 @@ XXXX-XX-XX **Bug fixes** -- 1193_: pids() may return False on OSX. +- 1193_: pids() may return False on macOS. 5.4.2 ===== @@ -112,7 +114,7 @@ XXXX-XX-XX - 1173_: introduced PSUTIL_DEBUG environment variable which can be set in order to print useful debug messages on stderr (useful in case of nasty errors). -- 1177_: added support for sensors_battery() on OSX. (patch by Arnon Yaari) +- 1177_: added support for sensors_battery() on macOS. (patch by Arnon Yaari) - 1183_: Process.children() is 2x faster on UNIX and 2.4x faster on Linux. - 1188_: deprecated method Process.memory_info_ex() now warns by using FutureWarning instead of DeprecationWarning. @@ -126,8 +128,8 @@ XXXX-XX-XX - 1179_: [Linux] Process.cmdline() is now able to splits cmdline args for misbehaving processes which overwrite /proc/pid/cmdline and use spaces instead of null bytes as args separator. -- 1181_: [OSX] Process.memory_maps() may raise ENOENT. -- 1187_: [OSX] pids() does not return PID 0 on recent OSX versions. +- 1181_: [macOS] Process.memory_maps() may raise ENOENT. +- 1187_: [macOS] pids() does not return PID 0 on recent macOS versions. 5.4.1 ===== @@ -164,7 +166,7 @@ XXXX-XX-XX - 1009_: [Linux] sensors_temperatures() may crash with IOError. - 1012_: [Windows] disk_io_counters()'s read_time and write_time were expressed in tens of micro seconds instead of milliseconds. -- 1127_: [OSX] invalid reference counting in Process.open_files() may lead to +- 1127_: [macOS] invalid reference counting in Process.open_files() may lead to segfault. (patch by Jakub Bacic) - 1129_: [Linux] sensors_fans() may crash with IOError. (patch by Sebastian Saip) @@ -236,14 +238,14 @@ XXXX-XX-XX cards installed. - 1021_: [Linux] open_files() may erroneously raise NoSuchProcess instead of skipping a file which gets deleted while open files are retrieved. -- 1029_: [OSX, FreeBSD] Process.connections('unix') on Python 3 doesn't +- 1029_: [macOS, FreeBSD] Process.connections('unix') on Python 3 doesn't properly handle unicode paths and may raise UnicodeDecodeError. -- 1033_: [OSX, FreeBSD] memory leak for net_connections() and +- 1033_: [macOS, FreeBSD] memory leak for net_connections() and Process.connections() when retrieving UNIX sockets (kind='unix'). - 1040_: fixed many unicode related issues such as UnicodeDecodeError on Python 3 + UNIX and invalid encoded data on Windows. - 1042_: [FreeBSD] psutil won't compile on FreeBSD 12. -- 1044_: [OSX] different Process methods incorrectly raise AccessDenied for +- 1044_: [macOS] different Process methods incorrectly raise AccessDenied for zombie processes. - 1046_: [Windows] disk_partitions() on Windows overrides user's SetErrorMode. - 1047_: [Windows] Process username(): memory leak in case exception is thrown. @@ -345,7 +347,8 @@ XXXX-XX-XX **Bug fixes** - 872_: [Linux] can now compile on Linux by using MUSL C library. -- 985_: [Windows] Fix a crash in `Process.open_files` when the worker thread for `NtQueryObject` times out. +- 985_: [Windows] Fix a crash in `Process.open_files` when the worker thread + for `NtQueryObject` times out. - 986_: [Linux] Process.cwd() may raise NoSuchProcess instead of ZombieProcess. 5.1.3 @@ -485,8 +488,8 @@ XXXX-XX-XX **Bug fixes** -- 514_: [OSX] possibly fix Process.memory_maps() segfault (critical!). -- 783_: [OSX] Process.status() may erroneously return "running" for zombie +- 514_: [macOS] possibly fix Process.memory_maps() segfault (critical!). +- 783_: [macOS] Process.status() may erroneously return "running" for zombie processes. - 798_: [Windows] Process.open_files() returns and empty list on Windows 10. - 825_: [Linux] cpu_affinity; fix possible double close and use of unopened @@ -499,12 +502,12 @@ XXXX-XX-XX - 906_: [BSD] disk_partitions(all=False) returned an empty list. Now the argument is ignored and all partitions are always returned. - 907_: [FreeBSD] Process.exe() may fail with OSError(ENOENT). -- 908_: [OSX, BSD] different process methods could errounesuly mask the real +- 908_: [macOS, BSD] different process methods could errounesuly mask the real error for high-privileged PIDs and raise NoSuchProcess and AccessDenied instead of OSError and RuntimeError. -- 909_: [OSX] Process open_files() and connections() methods may raise +- 909_: [macOS] Process open_files() and connections() methods may raise OSError with no exception set if process is gone. -- 916_: [OSX] fix many compilation warnings. +- 916_: [macOS] fix many compilation warnings. 4.3.1 ===== @@ -585,7 +588,7 @@ XXXX-XX-XX - 777_: [Linux] Process.open_files() on Linux return 3 new fields: position, mode and flags. - 779_: Process.cpu_times() returns two new fields, 'children_user' and - 'children_system' (always set to 0 on OSX and Windows). + 'children_system' (always set to 0 on macOS and Windows). - 789_: [Windows] psutil.cpu_times() return two new fields: "interrupt" and "dpc". Same for psutil.cpu_times_percent(). - 792_: new psutil.cpu_stats() function returning number of CPU ctx switches @@ -597,10 +600,10 @@ XXXX-XX-XX provides it. - 776_: [Linux] Process.cpu_affinity() may erroneously raise NoSuchProcess. (patch by wxwright) -- 780_: [OSX] psutil does not compile with some gcc versions. +- 780_: [macOS] psutil does not compile with some gcc versions. - 786_: net_if_addrs() may report incomplete MAC addresses. - 788_: [NetBSD] virtual_memory()'s buffers and shared values were set to 0. -- 790_: [OSX] psutil won't compile on OSX 10.4. +- 790_: [macOS] psutil won't compile on macOS 10.4. 4.0.0 ===== @@ -613,11 +616,11 @@ XXXX-XX-XX - 660_: [Windows] make.bat is smarter in finding alternative VS install locations. (patch by mpderbec) - 732_: Process.environ(). (patch by Frank Benkstein) -- 753_: [Linux, OSX, Windows] Process USS and PSS (Linux) "real" memory stats. +- 753_: [Linux, macOS, Windows] Process USS and PSS (Linux) "real" memory stats. (patch by Eric Rahm) - 755_: Process.memory_percent() "memtype" parameter. - 758_: tests now live in psutil namespace. -- 760_: expose OS constants (psutil.LINUX, psutil.OSX, etc.) +- 760_: expose OS constants (psutil.LINUX, psutil.macOS, etc.) - 756_: [Linux] disk_io_counters() return 2 new fields: read_merged_count and write_merged_count. - 762_: new scripts/procsmem.py script. @@ -734,7 +737,7 @@ XXXX-XX-XX - 644_: [Windows] added support for CTRL_C_EVENT and CTRL_BREAK_EVENT signals to use with Process.send_signal(). -- 648_: CI test integration for OSX. (patch by Jeff Tang) +- 648_: CI test integration for macOS. (patch by Jeff Tang) - 663_: [UNIX] net_if_addrs() now returns point-to-point (VPNs) addresses. - 655_: [Windows] different issues regarding unicode handling were fixed. On Python 2 all APIs returning a string will now return an encoded version of it @@ -1253,7 +1256,7 @@ DeprecationWarning. - 374_: [Windows] negative memory usage reported if process uses a lot of memory. - 379_: [Linux] Process.get_memory_maps() may raise ValueError. -- 394_: [OSX] Mapped memory regions report incorrect file name. +- 394_: [macOS] Mapped memory regions report incorrect file name. - 404_: [Linux] sched_*affinity() are implicitly declared. (patch by Arfrever) **API changes** @@ -1302,16 +1305,16 @@ DeprecationWarning. certain exotic Linux flavors having an incomplete /proc interface. If that's the case we now set the unretrievable stats to 0 and raise a RuntimeWarning. -- 315_: [OSX] fix some compilation warnings. +- 315_: [macOS] fix some compilation warnings. - 317_: [Windows] cannot set process CPU affinity above 31 cores. - 319_: [Linux] process get_memory_maps() raises KeyError 'Anonymous' on Debian squeeze. - 321_: [UNIX] Process.ppid property is no longer cached as the kernel may set the ppid to 1 in case of a zombie process. -- 323_: [OSX] disk_io_counters()'s read_time and write_time parameters were +- 323_: [macOS] disk_io_counters()'s read_time and write_time parameters were reporting microseconds not milliseconds. (patch by Gregory Szorc) - 331_: Process cmdline is no longer cached after first acces as it may change. -- 333_: [OSX] Leak of Mach ports on OS X (patch by rsesek@google.com) +- 333_: [macOS] Leak of Mach ports on macOS (patch by rsesek@google.com) - 337_: [Linux] process methods not working because of a poor /proc implementation will raise NotImplementedError rather than RuntimeError and Process.as_dict() will not blow up. (patch by Curtin1060) @@ -1324,7 +1327,7 @@ DeprecationWarning. - 338_: [Linux] disk_io_counters() fails to find some disks. - 351_: [Windows] if psutil is compiled with mingw32 (provided installers for py2.4 and py2.5 are) disk_io_counters() will fail. (Patch by m.malycha) -- 353_: [OSX] get_users() returns an empty list on OSX 10.8. +- 353_: [macOS] get_users() returns an empty list on macOS 10.8. - 356_: Process.parent now checks whether parent PID has been reused in which case returns None. - 365_: Process.set_nice() should check PID has not been reused by another @@ -1370,11 +1373,11 @@ DeprecationWarning. - 216_: [POSIX] get_connections() UNIX sockets support. - 220_: [FreeBSD] get_connections() has been rewritten in C and no longer requires lsof. -- 222_: [OSX] add support for process cwd. +- 222_: [macOS] add support for process cwd. - 261_: process extended memory info. -- 295_: [OSX] process executable path is now determined by asking the OS +- 295_: [macOS] process executable path is now determined by asking the OS instead of being guessed from process cmdline. -- 297_: [OSX] the Process methods below were always raising AccessDenied for +- 297_: [macOS] the Process methods below were always raising AccessDenied for any process except the current one. Now this is no longer true. Also they are 2.5x faster. - name @@ -1407,8 +1410,8 @@ DeprecationWarning. - active [POSIX] - inactive [POSIX] - buffers (BSD, Linux) - - cached (BSD, OSX) - - wired (OSX, BSD) + - cached (BSD, macOS) + - wired (macOS, BSD) - shared [FreeBSD] New psutil.swap_memory() provides: - total @@ -1425,7 +1428,7 @@ DeprecationWarning. **Bug fixes** -- 298_: [OSX and BSD] memory leak in get_num_fds(). +- 298_: [macOS and BSD] memory leak in get_num_fds(). - 299_: potential memory leak every time PyList_New(0) is used. - 303_: [Windows] potential heap corruption in get_num_threads() and get_status() Process methods. @@ -1480,7 +1483,7 @@ DeprecationWarning. - 245_: [POSIX] Process.wait() incrementally consumes less CPU cycles. - 257_: [Windows] removed Windows 2000 support. - 258_: [Linux] Process.get_memory_info() is now 0.5x faster. -- 260_: process's mapped memory regions. (Windows patch by wj32.64, OSX patch +- 260_: process's mapped memory regions. (Windows patch by wj32.64, macOS patch by Jeremy Whitlock) - 262_: [Windows] psutil.disk_partitions() was slow due to inspecting the floppy disk drive also when "all" argument was False. @@ -1503,7 +1506,7 @@ DeprecationWarning. - 193_: psutil.Popen constructor can throw an exception if the spawned process terminates quickly. -- 240_: [OSX] incorrect use of free() for Process.get_connections(). +- 240_: [macOS] incorrect use of free() for Process.get_connections(). - 244_: [POSIX] Process.wait() can hog CPU resources if called against a process which is not our children. - 248_: [Linux] psutil.network_io_counters() might return erroneous NIC names. @@ -1511,7 +1514,7 @@ DeprecationWarning. processes owned by another user. It now raises AccessDenied instead. - 266_: [Windows] psutil.get_pid_list() only shows 1024 processes. (patch by Amoser) -- 267_: [OSX] Process.get_connections() - an erroneous remote address was +- 267_: [macOS] Process.get_connections() - an erroneous remote address was returned. (Patch by Amoser) - 272_: [Linux] Porcess.get_open_files() - potential race condition can lead to unexpected NoSuchProcess exception. Also, we can get incorrect reports @@ -1542,7 +1545,7 @@ DeprecationWarning. **Bug fixes** - 228_: some example scripts were not working with python 3. -- 230_: [Windows / OSX] memory leak in Process.get_connections(). +- 230_: [Windows / macOS] memory leak in Process.get_connections(). - 232_: [Linux] psutil.phymem_usage() can report erroneous values which are different than "free" command. - 236_: [Windows] memory/handle leak in Process's get_memory_info(), @@ -1555,12 +1558,12 @@ DeprecationWarning. **Enhancements** -- 150_: network I/O counters. (OSX and Windows patch by Jeremy Whitlock) +- 150_: network I/O counters. (macOS and Windows patch by Jeremy Whitlock) - 154_: [FreeBSD] add support for process getcwd() - 157_: [Windows] provide installer for Python 3.2 64-bit. - 198_: Process.wait(timeout=0) can now be used to make wait() return immediately. -- 206_: disk I/O counters. (OSX and Windows patch by Jeremy Whitlock) +- 206_: disk I/O counters. (macOS and Windows patch by Jeremy Whitlock) - 213_: examples/iotop.py script. - 217_: Process.get_connections() now has a "kind" argument to filter for connections with different criteria. @@ -1571,7 +1574,7 @@ DeprecationWarning. **Bug fixes** -- 135_: [OSX] psutil cannot create Process object. +- 135_: [macOS] psutil cannot create Process object. - 144_: [Linux] no longer support 0 special PID. - 188_: [Linux] psutil import error on Linux ARM architectures. - 194_: [POSIX] psutil.Process.get_cpu_percent() now reports a percentage over @@ -1613,7 +1616,7 @@ DeprecationWarning. - 166_: get_memory_info() leaks handles hogging system resources. - 168_: psutil.cpu_percent() returns erroneous results when used in non-blocking mode. (patch by Philip Roberts) -- 178_: OSX - Process.get_threads() leaks memory +- 178_: macOS - Process.get_threads() leaks memory - 180_: [Windows] Process's get_num_threads() and get_threads() methods can raise NoSuchProcess exception while process still exists. @@ -1638,14 +1641,14 @@ DeprecationWarning. - 147_: per-process I/O nice (priority) - Linux only. - 148_: psutil.Popen class which tidies up subprocess.Popen and psutil.Process in a unique interface. -- 152_: [OSX] get_process_open_files() implementation has been rewritten +- 152_: [macOS] get_process_open_files() implementation has been rewritten in C and no longer relies on lsof resulting in a 3x speedup. -- 153_: [OSX] get_process_connection() implementation has been rewritten +- 153_: [macOS] get_process_connection() implementation has been rewritten in C and no longer relies on lsof resulting in a 3x speedup. **Bug fixes** -- 83_: process cmdline is empty on OSX 64-bit. +- 83_: process cmdline is empty on macOS 64-bit. - 130_: a race condition can cause IOError exception be raised on Linux if process disappears between open() and subsequent read() calls. - 145_: WindowsError was raised instead of psutil.AccessDenied when using @@ -1697,7 +1700,7 @@ DeprecationWarning. left behind every time Process class was instantiated. - 111_: path and name Process properties report truncated or erroneous values on UNIX. -- 120_: cpu_percent() always returning 100% on OS X. +- 120_: cpu_percent() always returning 100% on macOS. - 112_: uid and gid properties don't change if process changes effective user/group id at some point. - 126_: ppid, uid, gid, name, exe, cmdline and create_time properties are @@ -1746,7 +1749,7 @@ DeprecationWarning. available - 58_: is_running() is now called before kill() to make sure we are going to kill the correct process. -- 73_: virtual memory size reported on OS X includes shared library size +- 73_: virtual memory size reported on macOS includes shared library size - 77_: NoSuchProcess wasn't raised on Process.create_time if kill() was used first. @@ -1770,7 +1773,7 @@ DeprecationWarning. **Bug fixes** - 36_: [Windows] NoSuchProcess not raised when accessing timing methods. -- 40_: test_get_cpu_times() failing on FreeBSD and OS X. +- 40_: test_get_cpu_times() failing on FreeBSD and macOS. - 42_: [Windows] get_memory_percent() raises AccessDenied. 0.1.1 diff --git a/IDEAS b/IDEAS index 903726983..8190d48b9 100644 --- a/IDEAS +++ b/IDEAS @@ -18,7 +18,7 @@ PLATFORMS FEATURES ======== -- #371: sensors_temperatures() at least for OSX. +- #371: sensors_temperatures() at least for macOS. - #669: Windows / net_if_addrs(): return broadcast addr. @@ -27,7 +27,7 @@ FEATURES - #772: extended net_io_counters() metrics. -- #900: wheels for OSX and Linux. +- #900: wheels for macOS and Linux. - #922: extended net_io_stats() info. @@ -51,7 +51,7 @@ FEATURES Linux: yes Others: ? -- Process.threads(): thread names; patch for OSX available at: +- Process.threads(): thread names; patch for macOS available at: https://code.google.com/p/plcrashreporter/issues/detail?id=65 Sample code: https://github.com/janmojzis/pstree/blob/master/proc_kvm.c diff --git a/INSTALL.rst b/INSTALL.rst index 4ceb0c19b..6644fb37b 100644 --- a/INSTALL.rst +++ b/INSTALL.rst @@ -10,7 +10,7 @@ On Linux or via wget: wget https://bootstrap.pypa.io/get-pip.py -O - | python -On OSX or via curl: +On macOS or via curl: .. code-block:: bash @@ -61,8 +61,8 @@ RedHat / CentOS: If you're on Python 3 use ``python3-dev`` and ``python3-pip`` instead. -OSX -=== +macOS +===== Install `Xcode `__ first, then: diff --git a/MANIFEST.in b/MANIFEST.in index 26d678fc3..c6d8c550a 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -28,7 +28,7 @@ include psutil/_exceptions.py include psutil/_psaix.py include psutil/_psbsd.py include psutil/_pslinux.py -include psutil/_psosx.py +include psutil/_psmacos.py include psutil/_psposix.py include psutil/_pssunos.py include psutil/_psutil_aix.c @@ -36,7 +36,7 @@ include psutil/_psutil_bsd.c include psutil/_psutil_common.c include psutil/_psutil_common.h include psutil/_psutil_linux.c -include psutil/_psutil_osx.c +include psutil/_psutil_macos.c include psutil/_psutil_posix.c include psutil/_psutil_posix.h include psutil/_psutil_sunos.c @@ -55,14 +55,16 @@ include psutil/arch/freebsd/specific.c include psutil/arch/freebsd/specific.h include psutil/arch/freebsd/sys_socks.c include psutil/arch/freebsd/sys_socks.h +include psutil/arch/macos/process_info.c +include psutil/arch/macos/process_info.h +include psutil/arch/macos/smc.c +include psutil/arch/macos/smc.h include psutil/arch/netbsd/socks.c include psutil/arch/netbsd/socks.h include psutil/arch/netbsd/specific.c include psutil/arch/netbsd/specific.h include psutil/arch/openbsd/specific.c include psutil/arch/openbsd/specific.h -include psutil/arch/osx/process_info.c -include psutil/arch/osx/process_info.h include psutil/arch/solaris/environ.c include psutil/arch/solaris/environ.h include psutil/arch/solaris/v10/ifaddrs.c @@ -86,9 +88,9 @@ include psutil/tests/test_bsd.py include psutil/tests/test_connections.py include psutil/tests/test_contracts.py include psutil/tests/test_linux.py +include psutil/tests/test_macos.py include psutil/tests/test_memory_leaks.py include psutil/tests/test_misc.py -include psutil/tests/test_osx.py include psutil/tests/test_posix.py include psutil/tests/test_process.py include psutil/tests/test_sunos.py diff --git a/Makefile b/Makefile index 51047383f..58c0d3961 100644 --- a/Makefile +++ b/Makefile @@ -142,7 +142,7 @@ test-posix: ## POSIX specific tests. test-platform: ## Run specific platform tests only. ${MAKE} install - $(TEST_PREFIX) $(PYTHON) psutil/tests/test_`$(PYTHON) -c 'import psutil; print([x.lower() for x in ("LINUX", "BSD", "OSX", "SUNOS", "WINDOWS", "AIX") if getattr(psutil, x)][0])'`.py + $(TEST_PREFIX) $(PYTHON) psutil/tests/test_`$(PYTHON) -c 'import psutil; print([x.lower() for x in ("LINUX", "BSD", "MACOS", "SUNOS", "WINDOWS", "AIX") if getattr(psutil, x)][0])'`.py test-memleaks: ## Memory leak tests. ${MAKE} install diff --git a/README.rst b/README.rst index 0852b1ce5..abd5816c6 100644 --- a/README.rst +++ b/README.rst @@ -1,4 +1,4 @@ -.. image:: https://img.shields.io/travis/giampaolo/psutil/master.svg?maxAge=3600&label=Linux%20/%20OSX +.. image:: https://img.shields.io/travis/giampaolo/psutil/master.svg?maxAge=3600&label=Linux%20/%20macOS :target: https://travis-ci.org/giampaolo/psutil :alt: Linux tests (Travis) @@ -56,7 +56,7 @@ psutil currently supports the following platforms: - **Linux** - **Windows** -- **OSX**, +- **macOS**, - **FreeBSD, OpenBSD**, **NetBSD** - **Sun Solaris** - **AIX** @@ -307,7 +307,7 @@ Process management >>> >>> p.memory_info() pmem(rss=10915840, vms=67608576, shared=3313664, text=2310144, lib=0, data=7262208, dirty=0) - >>> p.memory_full_info() # "real" USS memory usage (Linux, OSX, Win only) + >>> p.memory_full_info() # "real" USS memory usage (Linux, macOS, Win only) pfullmem(rss=10199040, vms=52133888, shared=3887104, text=2867200, lib=0, data=5967872, dirty=0, uss=6545408, pss=6872064, swap=0) >>> p.memory_percent() 0.7823 diff --git a/docs/index.rst b/docs/index.rst index 208f59e9c..47167d764 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -32,7 +32,7 @@ psutil currently supports the following platforms: - **Linux** - **Windows** -- **OSX**, +- **macOS**, - **FreeBSD, OpenBSD**, **NetBSD** - **Sun Solaris** - **AIX** @@ -235,7 +235,7 @@ CPU scpufreq(current=1703.609, min=800.0, max=3500.0), scpufreq(current=1754.289, min=800.0, max=3500.0)] - Availability: Linux, OSX, Windows + Availability: Linux, macOS, Windows .. versionadded:: 5.1.0 @@ -272,7 +272,7 @@ Memory - **shared** *(Linux, BSD)*: memory that may be simultaneously accessed by multiple processes. - **slab** *(Linux)*: in-kernel data structures cache. - - **wired** *(BSD, OSX)*: memory that is marked to always stay in RAM. It is + - **wired** *(BSD, macOS)*: memory that is marked to always stay in RAM. It is never moved to disk. The sum of **used** and **available** does not necessarily equal **total**. @@ -345,7 +345,7 @@ Disks On Windows it is determined via `GetDriveType `__ and can be either ``"removable"``, ``"fixed"``, ``"remote"``, ``"cdrom"``, - ``"unmounted"`` or ``"ramdisk"``. On OSX and BSD it is retrieved via + ``"unmounted"`` or ``"ramdisk"``. On macOS and BSD it is retrieved via `getfsstat(2) `__. See `disk_usage.py `__ script providing an example usage. @@ -462,7 +462,7 @@ Network - **errout**: total number of errors while sending - **dropin**: total number of incoming packets which were dropped - **dropout**: total number of outgoing packets which were dropped (always 0 - on OSX and BSD) + on macOS and BSD) If *pernic* is ``True`` return the same information for every network interface installed on the system as a dictionary with network interface @@ -555,7 +555,7 @@ Network | ``"all"`` | the sum of all the possible families and protocols | +----------------+-----------------------------------------------------+ - On OSX and AIX this function requires root privileges. + On macOS and AIX this function requires root privileges. To get per-process connections use :meth:`Process.connections`. Also, see `netstat.py sample script `__. @@ -570,7 +570,7 @@ Network ...] .. note:: - (OSX and AIX) :class:`psutil.AccessDenied` is always raised unless running + (macOS and AIX) :class:`psutil.AccessDenied` is always raised unless running as root. This is a limitation of the OS and ``lsof`` does the same. .. note:: @@ -698,11 +698,11 @@ Sensors See also `temperatures.py `__ and `sensors.py `__ for an example application. - Availability: Linux, OSX + Availability: Linux, macOS .. versionadded:: 5.1.0 - .. versionchanged:: 5.5.0: added OSX support + .. versionchanged:: 5.5.0: added macOS support .. warning:: @@ -724,11 +724,11 @@ Sensors See also `fans.py `__ and `sensors.py `__ for an example application. - Availability: Linux, OSX + Availability: Linux, macOS .. versionadded:: 5.2.0 - .. versionchanged:: 5.5.0: added OSX support + .. versionchanged:: 5.5.0: added macOS support .. warning:: @@ -772,7 +772,7 @@ Sensors .. versionadded:: 5.1.0 - .. versionchanged:: 5.4.2 added OSX support + .. versionchanged:: 5.4.2 added macOS support .. warning:: @@ -1070,7 +1070,7 @@ Process class if you call all the methods together (best case scenario). +------------------------------+-------------------------------+------------------------------+------------------------------+--------------------------+--------------------------+ - | Linux | Windows | OSX | BSD | SunOS | AIX | + | Linux | Windows | macOS | BSD | SunOS | AIX | +==============================+===============================+==============================+==============================+==========================+==========================+ | :meth:`cpu_num` | :meth:`cpu_percent` | :meth:`cpu_percent` | :meth:`cpu_num` | :meth:`name` | :meth:`name` | +------------------------------+-------------------------------+------------------------------+------------------------------+--------------------------+--------------------------+ @@ -1158,7 +1158,7 @@ Process class >>> psutil.Process().environ() {'LC_NUMERIC': 'it_IT.UTF-8', 'QT_QPA_PLATFORMTHEME': 'appmenu-qt5', 'IM_CONFIG_PHASE': '1', 'XDG_GREETER_DATA_DIR': '/var/lib/lightdm-data/giampaolo', 'GNOME_DESKTOP_SESSION_ID': 'this-is-deprecated', 'XDG_CURRENT_DESKTOP': 'Unity', 'UPSTART_EVENTS': 'started starting', 'GNOME_KEYRING_PID': '', 'XDG_VTNR': '7', 'QT_IM_MODULE': 'ibus', 'LOGNAME': 'giampaolo', 'USER': 'giampaolo', 'PATH': '/home/giampaolo/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin:/home/giampaolo/svn/sysconf/bin', 'LC_PAPER': 'it_IT.UTF-8', 'GNOME_KEYRING_CONTROL': '', 'GTK_IM_MODULE': 'ibus', 'DISPLAY': ':0', 'LANG': 'en_US.UTF-8', 'LESS_TERMCAP_se': '\x1b[0m', 'TERM': 'xterm-256color', 'SHELL': '/bin/bash', 'XDG_SESSION_PATH': '/org/freedesktop/DisplayManager/Session0', 'XAUTHORITY': '/home/giampaolo/.Xauthority', 'LANGUAGE': 'en_US', 'COMPIZ_CONFIG_PROFILE': 'ubuntu', 'LC_MONETARY': 'it_IT.UTF-8', 'QT_LINUX_ACCESSIBILITY_ALWAYS_ON': '1', 'LESS_TERMCAP_me': '\x1b[0m', 'LESS_TERMCAP_md': '\x1b[01;38;5;74m', 'LESS_TERMCAP_mb': '\x1b[01;31m', 'HISTSIZE': '100000', 'UPSTART_INSTANCE': '', 'CLUTTER_IM_MODULE': 'xim', 'WINDOWID': '58786407', 'EDITOR': 'vim', 'SESSIONTYPE': 'gnome-session', 'XMODIFIERS': '@im=ibus', 'GPG_AGENT_INFO': '/home/giampaolo/.gnupg/S.gpg-agent:0:1', 'HOME': '/home/giampaolo', 'HISTFILESIZE': '100000', 'QT4_IM_MODULE': 'xim', 'GTK2_MODULES': 'overlay-scrollbar', 'XDG_SESSION_DESKTOP': 'ubuntu', 'SHLVL': '1', 'XDG_RUNTIME_DIR': '/run/user/1000', 'INSTANCE': 'Unity', 'LC_ADDRESS': 'it_IT.UTF-8', 'SSH_AUTH_SOCK': '/run/user/1000/keyring/ssh', 'VTE_VERSION': '4205', 'GDMSESSION': 'ubuntu', 'MANDATORY_PATH': '/usr/share/gconf/ubuntu.mandatory.path', 'VISUAL': 'vim', 'DESKTOP_SESSION': 'ubuntu', 'QT_ACCESSIBILITY': '1', 'XDG_SEAT_PATH': '/org/freedesktop/DisplayManager/Seat0', 'LESSCLOSE': '/usr/bin/lesspipe %s %s', 'LESSOPEN': '| /usr/bin/lesspipe %s', 'XDG_SESSION_ID': 'c2', 'DBUS_SESSION_BUS_ADDRESS': 'unix:abstract=/tmp/dbus-9GAJpvnt8r', '_': '/usr/bin/python', 'DEFAULTS_PATH': '/usr/share/gconf/ubuntu.default.path', 'LC_IDENTIFICATION': 'it_IT.UTF-8', 'LESS_TERMCAP_ue': '\x1b[0m', 'UPSTART_SESSION': 'unix:abstract=/com/ubuntu/upstart-session/1000/1294', 'XDG_CONFIG_DIRS': '/etc/xdg/xdg-ubuntu:/usr/share/upstart/xdg:/etc/xdg', 'GTK_MODULES': 'gail:atk-bridge:unity-gtk-module', 'XDG_SESSION_TYPE': 'x11', 'PYTHONSTARTUP': '/home/giampaolo/.pythonstart', 'LC_NAME': 'it_IT.UTF-8', 'OLDPWD': '/home/giampaolo/svn/curio_giampaolo/tests', 'GDM_LANG': 'en_US', 'LC_TELEPHONE': 'it_IT.UTF-8', 'HISTCONTROL': 'ignoredups:erasedups', 'LC_MEASUREMENT': 'it_IT.UTF-8', 'PWD': '/home/giampaolo/svn/curio_giampaolo', 'JOB': 'gnome-session', 'LESS_TERMCAP_us': '\x1b[04;38;5;146m', 'UPSTART_JOB': 'unity-settings-daemon', 'LC_TIME': 'it_IT.UTF-8', 'LESS_TERMCAP_so': '\x1b[38;5;246m', 'PAGER': 'less', 'XDG_DATA_DIRS': '/usr/share/ubuntu:/usr/share/gnome:/usr/local/share/:/usr/share/:/var/lib/snapd/desktop', 'XDG_SEAT': 'seat0'} - Availability: Linux, OSX, Windows, SunOS + Availability: Linux, macOS, Windows, SunOS .. versionadded:: 4.0.0 .. versionchanged:: 5.3.0 added SunOS support @@ -1414,7 +1414,7 @@ Process class Return a `(user, system, children_user, children_system)` named tuple representing the accumulated process time, in seconds (see `explanation `__). - On Windows and OSX only *user* and *system* are filled, the others are + On Windows and macOS only *user* and *system* are filled, the others are set to ``0``. This is similar to `os.times() `__ @@ -1522,7 +1522,7 @@ Process class All numbers are expressed in bytes. +---------+---------+-------+---------+-----+------------------------------+ - | Linux | OSX | BSD | Solaris | AIX | Windows | + | Linux | macOS | BSD | Solaris | AIX | Windows | +=========+=========+=======+=========+=====+==============================+ | rss | rss | rss | rss | rss | rss (alias for ``wset``) | +---------+---------+-------+---------+-----+------------------------------+ @@ -1582,9 +1582,9 @@ Process class - **dirty** *(Linux)*: the number of dirty pages. - - **pfaults** *(OSX)*: number of page faults. + - **pfaults** *(macOS)*: number of page faults. - - **pageins** *(OSX)*: number of actual pageins. + - **pageins** *(macOS)*: number of actual pageins. For on explanation of Windows fields rely on `PROCESS_MEMORY_COUNTERS_EX `__ structure doc. @@ -1608,7 +1608,7 @@ Process class .. method:: memory_full_info() This method returns the same information as :meth:`memory_info`, plus, on - some platform (Linux, OSX, Windows), also provides additional metrics + some platform (Linux, macOS, Windows), also provides additional metrics (USS, PSS and swap). The additional metrics provide a better representation of "effective" process memory consumption (in case of USS) as explained in detail in this @@ -1619,7 +1619,7 @@ Process class On platforms where extra fields are not implemented this simply returns the same metrics as :meth:`memory_info`. - - **uss** *(Linux, OSX, Windows)*: + - **uss** *(Linux, macOS, Windows)*: aka "Unique Set Size", this is the memory which is unique to a process and which would be freed if the process was terminated right now. @@ -1678,7 +1678,7 @@ Process class for an example application. +---------------+--------------+---------+-----------+--------------+ - | Linux | OSX | Windows | Solaris | FreeBSD | + | Linux | macOS | Windows | Solaris | FreeBSD | +===============+==============+=========+===========+==============+ | rss | rss | rss | rss | rss | +---------------+--------------+---------+-----------+--------------+ @@ -2118,7 +2118,7 @@ Constants .. data:: POSIX .. data:: WINDOWS .. data:: LINUX -.. data:: OSX +.. data:: MACOS .. data:: FREEBSD .. data:: NETBSD .. data:: OPENBSD @@ -2132,6 +2132,13 @@ Constants .. versionadded:: 4.0.0 .. versionchanged:: 5.4.0 added AIX +.. data:: OSX + + Alias for :const:`MACOS` (deprecated). + + .. warning:: + deprecated in version 5.5.0; use :const:`MACOS` instead. + .. _const-procfs_path: .. data:: PROCFS_PATH @@ -2164,7 +2171,7 @@ Constants .. data:: STATUS_DEAD .. data:: STATUS_WAKE_KILL .. data:: STATUS_WAKING -.. data:: STATUS_IDLE (OSX, FreeBSD) +.. data:: STATUS_IDLE (macOS, FreeBSD) .. data:: STATUS_LOCKED (FreeBSD) .. data:: STATUS_WAITING (FreeBSD) .. data:: STATUS_SUSPENDED (NetBSD) @@ -2320,7 +2327,7 @@ methods such as :meth:`Process.username` or :meth:`WindowsService.description`: * all strings are encoded by using the OS filesystem encoding (``sys.getfilesystemencoding()``) which varies depending on the platform - (e.g. "UTF-8" on OSX, "mbcs" on Win) + (e.g. "UTF-8" on macOS, "mbcs" on Win) * no API call is supposed to crash with ``UnicodeDecodeError`` * instead, in case of badly encoded data returned by the OS, the following error handlers are used to replace the corrupted characters in the string: * Python 3: ``sys.getfilesystemencodeerrors()`` (PY 3.6+) or @@ -2589,7 +2596,7 @@ FAQs * Q: Why do I get :class:`AccessDenied` for certain processes? * A: This may happen when you query processess owned by another user, - especially on `OSX `__ and + especially on `macOS `__ and Windows. Unfortunately there's not much you can do about this except running the Python process with higher privileges. diff --git a/psutil/__init__.py b/psutil/__init__.py index f38592792..affc5c8c3 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -10,7 +10,7 @@ - Linux - Windows - - OSX + - macOS - FreeBSD - OpenBSD - NetBSD @@ -78,9 +78,10 @@ from ._common import BSD from ._common import FREEBSD # NOQA from ._common import LINUX +from ._common import MACOS from ._common import NETBSD # NOQA from ._common import OPENBSD # NOQA -from ._common import OSX +from ._common import OSX # deprecated alias from ._common import POSIX # NOQA from ._common import SUNOS from ._common import WINDOWS @@ -151,8 +152,8 @@ from ._psutil_windows import REALTIME_PRIORITY_CLASS # NOQA from ._pswindows import CONN_DELETE_TCB # NOQA -elif OSX: - from . import _psosx as _psplatform +elif MACOS: + from . import _psmacos as _psplatform elif BSD: from . import _psbsd as _psplatform @@ -199,8 +200,8 @@ "POWER_TIME_UNKNOWN", "POWER_TIME_UNLIMITED", - "BSD", "FREEBSD", "LINUX", "NETBSD", "OPENBSD", "OSX", "POSIX", "SUNOS", - "WINDOWS", "AIX", + "BSD", "FREEBSD", "LINUX", "NETBSD", "OPENBSD", "MACOS", "OSX", "POSIX", + "SUNOS", "WINDOWS", "AIX", # classes "Process", "Popen", @@ -823,7 +824,7 @@ def cpu_num(self): """ return self._proc.cpu_num() - # Linux, OSX and Windows only + # Linux, macOS and Windows only if hasattr(_psplatform.Process, "environ"): def environ(self): @@ -1027,7 +1028,7 @@ def cpu_times(self): namedtuple representing the accumulated process time, in seconds. This is similar to os.times() but per-process. - On OSX and Windows children_user and children_system are + On macOS and Windows children_user and children_system are always set to 0. """ return self._proc.cpu_times() @@ -1049,7 +1050,7 @@ def memory_info_ex(self): def memory_full_info(self): """This method returns the same information as memory_info(), - plus, on some platform (Linux, OSX, Windows), also provides + plus, on some platform (Linux, macOS, Windows), also provides additional metrics (USS, PSS and swap). The additional metrics provide a better representation of actual process memory usage. @@ -1898,9 +1899,9 @@ def virtual_memory(): - used: memory used, calculated differently depending on the platform and designed for informational purposes only: - OSX: active + inactive + wired + macOS: active + inactive + wired BSD: active + wired + cached - LINUX: total - free + Linux: total - free - free: memory not being used at all (zeroed) that is readily available; @@ -1918,10 +1919,10 @@ def virtual_memory(): - buffers (BSD, Linux): cache for things like file system metadata. - - cached (BSD, OSX): + - cached (BSD, macOS): cache for various things. - - wired (OSX, BSD): + - wired (macOS, BSD): memory that is marked to always stay in RAM. It is never moved to disk. - shared (BSD): @@ -2046,7 +2047,7 @@ def net_io_counters(pernic=False, nowrap=True): - errout: total number of errors while sending - dropin: total number of incoming packets which were dropped - dropout: total number of outgoing packets which were dropped - (always 0 on OSX and BSD) + (always 0 on macOS and BSD) If *pernic* is True return the same information for every network interface installed on the system as a dictionary @@ -2102,7 +2103,7 @@ def net_connections(kind='inet'): | all | the sum of all the possible families and protocols | +------------+----------------------------------------------------+ - On OSX this function requires root privileges. + On macOS this function requires root privileges. """ return _psplatform.net_connections(kind) @@ -2175,7 +2176,7 @@ def net_if_stats(): # ===================================================================== -# Linux, OSX +# Linux, macOS if hasattr(_psplatform, "sensors_temperatures"): def sensors_temperatures(fahrenheit=False): @@ -2213,7 +2214,7 @@ def convert(n): __all__.append("sensors_temperatures") -# Linux, OSX +# Linux, macOS if hasattr(_psplatform, "sensors_fans"): def sensors_fans(): @@ -2226,7 +2227,7 @@ def sensors_fans(): __all__.append("sensors_fans") -# Linux, Windows, FreeBSD, OSX +# Linux, Windows, FreeBSD, macOS if hasattr(_psplatform, "sensors_battery"): def sensors_battery(): diff --git a/psutil/_common.py b/psutil/_common.py index af1fb563c..6fcf8fe05 100644 --- a/psutil/_common.py +++ b/psutil/_common.py @@ -42,8 +42,8 @@ __all__ = [ # constants - 'FREEBSD', 'BSD', 'LINUX', 'NETBSD', 'OPENBSD', 'OSX', 'POSIX', 'SUNOS', - 'WINDOWS', + 'FREEBSD', 'BSD', 'LINUX', 'NETBSD', 'OPENBSD', 'MACOS', 'OSX', 'POSIX', + 'SUNOS', 'WINDOWS', 'ENCODING', 'ENCODING_ERRS', 'AF_INET6', # connection constants 'CONN_CLOSE', 'CONN_CLOSE_WAIT', 'CONN_CLOSING', 'CONN_ESTABLISHED', @@ -75,7 +75,8 @@ POSIX = os.name == "posix" WINDOWS = os.name == "nt" LINUX = sys.platform.startswith("linux") -OSX = sys.platform.startswith("darwin") +MACOS = sys.platform.startswith("darwin") +OSX = MACOS # deprecated alias FREEBSD = sys.platform.startswith("freebsd") OPENBSD = sys.platform.startswith("openbsd") NETBSD = sys.platform.startswith("netbsd") @@ -99,7 +100,7 @@ STATUS_DEAD = "dead" STATUS_WAKE_KILL = "wake-kill" STATUS_WAKING = "waking" -STATUS_IDLE = "idle" # FreeBSD, OSX +STATUS_IDLE = "idle" # FreeBSD, macOS STATUS_LOCKED = "locked" # FreeBSD STATUS_WAITING = "waiting" # FreeBSD STATUS_SUSPENDED = "suspended" # NetBSD diff --git a/psutil/_exceptions.py b/psutil/_exceptions.py index c08e6d83c..6dbbd2826 100644 --- a/psutil/_exceptions.py +++ b/psutil/_exceptions.py @@ -39,7 +39,7 @@ def __init__(self, pid, name=None, msg=None): class ZombieProcess(NoSuchProcess): """Exception raised when querying a zombie process. This is - raised on OSX, BSD and Solaris only, and not always: depending + raised on macOS, BSD and Solaris only, and not always: depending on the query the OS may be able to succeed anyway. On Linux all zombie processes are querable (hence this is never raised). Windows doesn't have zombie processes. diff --git a/psutil/_psosx.py b/psutil/_psmacos.py similarity index 98% rename from psutil/_psosx.py rename to psutil/_psmacos.py index a0101df07..a719bedaa 100644 --- a/psutil/_psosx.py +++ b/psutil/_psmacos.py @@ -2,7 +2,7 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -"""OSX platform implementation.""" +"""macOS platform implementation.""" import collections import contextlib @@ -14,7 +14,7 @@ from . import _common from . import _psposix -from . import _psutil_osx as cext +from . import _psutil_macos as cext from . import _psutil_posix as cext_posix from ._common import AF_INET6 from ._common import conn_tmap @@ -277,7 +277,7 @@ def cpu_stats(): def cpu_freq(): """Return CPU frequency. - On OSX per-cpu frequency is not supported. + On macOS per-cpu frequency is not supported. Also, the returned frequency never changes, see: https://arstechnica.com/civis/viewtopic.php?f=19&t=465002 """ @@ -381,7 +381,7 @@ def sensors_fans(): def net_connections(kind='inet'): """System-wide network connections.""" - # Note: on OSX this will fail with AccessDenied unless + # Note: on macOS this will fail with AccessDenied unless # the process is owned by root. ret = [] for pid in pids(): @@ -444,7 +444,7 @@ def users(): def pids(): ls = cext.pids() if 0 not in ls: - # On certain OSX versions pids() C doesn't return PID 0 but + # On certain macOS versions pids() C doesn't return PID 0 but # "ps" does and the process is querable via sysctl(): # https://travis-ci.org/giampaolo/psutil/jobs/309619941 try: diff --git a/psutil/_psutil_bsd.c b/psutil/_psutil_bsd.c index 9a2ed04bc..4e91c02ed 100644 --- a/psutil/_psutil_bsd.c +++ b/psutil/_psutil_bsd.c @@ -393,7 +393,7 @@ psutil_proc_cmdline(PyObject *self, PyObject *args) { /* * Return the number of logical CPUs in the system. - * XXX this could be shared with OSX + * XXX this could be shared with macOS */ static PyObject * psutil_cpu_count_logical(PyObject *self, PyObject *args) { diff --git a/psutil/_psutil_osx.c b/psutil/_psutil_macos.c similarity index 98% rename from psutil/_psutil_osx.c rename to psutil/_psutil_macos.c index 15f4d4f69..8fc3140f6 100644 --- a/psutil/_psutil_osx.c +++ b/psutil/_psutil_macos.c @@ -3,7 +3,7 @@ * Use of this source code is governed by a BSD-style license that can be * found in the LICENSE file. * - * OS X platform-specific module methods for _psutil_osx + * macOS platform-specific module methods. */ #include @@ -43,8 +43,8 @@ #include "_psutil_common.h" #include "_psutil_posix.h" -#include "arch/osx/process_info.h" -#include "arch/osx/smc.h" +#include "arch/macos/process_info.h" +#include "arch/macos/smc.h" #define PSUTIL_TV2DOUBLE(t) ((t).tv_sec + (t).tv_usec / 1000000.0) @@ -251,7 +251,7 @@ psutil_proc_pidtaskinfo_oneshot(PyObject *self, PyObject *args) { "(ddKKkkkk)", (float)pti.pti_total_user / 1000000000.0, // (float) cpu user time (float)pti.pti_total_system / 1000000000.0, // (float) cpu sys time - // Note about memory: determining other mem stats on OSX is a mess: + // Note about memory: determining other mem stats on macOS is a mess: // http://www.opensource.apple.com/source/top/top-67/libtop.c?txt // I just give up. // struct proc_regioninfo pri; @@ -342,7 +342,7 @@ psutil_proc_cmdline(PyObject *self, PyObject *args) { if (! PyArg_ParseTuple(args, "l", &pid)) return NULL; - // get the commandline, defined in arch/osx/process_info.c + // get the commandline, defined in arch/macos/process_info.c py_retlist = psutil_get_cmdline(pid); return py_retlist; } @@ -359,7 +359,7 @@ psutil_proc_environ(PyObject *self, PyObject *args) { if (! PyArg_ParseTuple(args, "l", &pid)) return NULL; - // get the environment block, defined in arch/osx/process_info.c + // get the environment block, defined in arch/macos/process_info.c py_retdict = psutil_get_environ(pid); return py_retdict; } @@ -1759,7 +1759,7 @@ psutil_disk_io_counters(PyObject *self, PyObject *args) { CFNumberGetValue(number, kCFNumberSInt64Type, &write_time); } - // Read/Write time on OS X comes back in nanoseconds and in psutil + // Read/Write time on macOS comes back in nanoseconds and in psutil // we've standardized on milliseconds so do the conversion. py_disk_info = Py_BuildValue( "(KKKKKK)", @@ -2066,13 +2066,13 @@ struct module_state { #if PY_MAJOR_VERSION >= 3 static int -psutil_osx_traverse(PyObject *m, visitproc visit, void *arg) { +psutil_macos_traverse(PyObject *m, visitproc visit, void *arg) { Py_VISIT(GETSTATE(m)->error); return 0; } static int -psutil_osx_clear(PyObject *m) { +psutil_macos_clear(PyObject *m) { Py_CLEAR(GETSTATE(m)->error); return 0; } @@ -2080,31 +2080,31 @@ psutil_osx_clear(PyObject *m) { static struct PyModuleDef moduledef = { PyModuleDef_HEAD_INIT, - "psutil_osx", + "psutil_macos", NULL, sizeof(struct module_state), PsutilMethods, NULL, - psutil_osx_traverse, - psutil_osx_clear, + psutil_macos_traverse, + psutil_macos_clear, NULL }; #define INITERROR return NULL -PyMODINIT_FUNC PyInit__psutil_osx(void) +PyMODINIT_FUNC PyInit__psutil_macos(void) #else #define INITERROR return void -init_psutil_osx(void) +init_psutil_macos(void) #endif { #if PY_MAJOR_VERSION >= 3 PyObject *module = PyModule_Create(&moduledef); #else - PyObject *module = Py_InitModule("_psutil_osx", PsutilMethods); + PyObject *module = Py_InitModule("_psutil_macos", PsutilMethods); #endif PyModule_AddIntConstant(module, "version", PSUTIL_VERSION); // process status constants, defined in: @@ -2130,7 +2130,7 @@ init_psutil_osx(void) // Exception. ZombieProcessError = PyErr_NewException( - "_psutil_osx.ZombieProcessError", NULL, NULL); + "_psutil_macos.ZombieProcessError", NULL, NULL); Py_INCREF(ZombieProcessError); PyModule_AddObject(module, "ZombieProcessError", ZombieProcessError); diff --git a/psutil/_psutil_posix.c b/psutil/_psutil_posix.c index cc827273c..e9a8f2d95 100644 --- a/psutil/_psutil_posix.c +++ b/psutil/_psutil_posix.c @@ -28,7 +28,7 @@ #include #include #include -#elif defined(PSUTIL_BSD) || defined(PSUTIL_OSX) +#elif defined(PSUTIL_BSD) || defined(PSUTIL_MACOS) #include #include #include @@ -71,7 +71,7 @@ psutil_pid_exists(long pid) { #endif } -#if defined(PSUTIL_OSX) +#if defined(PSUTIL_MACOS) ret = kill((pid_t)pid , 0); #else ret = kill(pid , 0); @@ -143,7 +143,7 @@ psutil_posix_getpriority(PyObject *self, PyObject *args) { if (! PyArg_ParseTuple(args, "l", &pid)) return NULL; -#ifdef PSUTIL_OSX +#ifdef PSUTIL_MACOS priority = getpriority(PRIO_PROCESS, (id_t)pid); #else priority = getpriority(PRIO_PROCESS, pid); @@ -166,7 +166,7 @@ psutil_posix_setpriority(PyObject *self, PyObject *args) { if (! PyArg_ParseTuple(args, "li", &pid, &priority)) return NULL; -#ifdef PSUTIL_OSX +#ifdef PSUTIL_MACOS retval = setpriority(PRIO_PROCESS, (id_t)pid, priority); #else retval = setpriority(PRIO_PROCESS, pid, priority); @@ -221,7 +221,7 @@ psutil_convert_ipaddr(struct sockaddr *addr, int family) { len = lladdr->sll_halen; data = (const char *)lladdr->sll_addr; } -#elif defined(PSUTIL_BSD) || defined(PSUTIL_OSX) +#elif defined(PSUTIL_BSD) || defined(PSUTIL_MACOS) else if (addr->sa_family == AF_LINK) { // Note: prior to Python 3.4 socket module does not expose // AF_LINK so we'll do. @@ -431,9 +431,9 @@ psutil_net_if_flags(PyObject *self, PyObject *args) { /* - * net_if_stats() OSX/BSD implementation. + * net_if_stats() macOS/BSD implementation. */ -#if defined(PSUTIL_BSD) || defined(PSUTIL_OSX) +#if defined(PSUTIL_BSD) || defined(PSUTIL_MACOS) int psutil_get_nic_speed(int ifm_active) { // Determine NIC speed. Taken from: @@ -620,7 +620,7 @@ psutil_net_if_duplex_speed(PyObject *self, PyObject *args) { close(sock); return PyErr_SetFromErrno(PyExc_OSError); } -#endif // net_if_stats() OSX/BSD implementation +#endif // net_if_stats() macOS/BSD implementation /* @@ -638,7 +638,7 @@ PsutilMethods[] = { "Retrieve NIC MTU"}, {"net_if_flags", psutil_net_if_flags, METH_VARARGS, "Retrieve NIC flags"}, -#if defined(PSUTIL_BSD) || defined(PSUTIL_OSX) +#if defined(PSUTIL_BSD) || defined(PSUTIL_MACOS) {"net_if_duplex_speed", psutil_net_if_duplex_speed, METH_VARARGS, "Return NIC stats."}, #endif @@ -698,7 +698,7 @@ void init_psutil_posix(void) PyObject *module = Py_InitModule("_psutil_posix", PsutilMethods); #endif -#if defined(PSUTIL_BSD) || defined(PSUTIL_OSX) || defined(PSUTIL_SUNOS) || defined(PSUTIL_AIX) +#if defined(PSUTIL_BSD) || defined(PSUTIL_MACOS) || defined(PSUTIL_SUNOS) || defined(PSUTIL_AIX) PyModule_AddIntConstant(module, "AF_LINK", AF_LINK); #endif diff --git a/psutil/arch/osx/process_info.c b/psutil/arch/macos/process_info.c similarity index 98% rename from psutil/arch/osx/process_info.c rename to psutil/arch/macos/process_info.c index 40c79a2cd..de007de48 100644 --- a/psutil/arch/osx/process_info.c +++ b/psutil/arch/macos/process_info.c @@ -4,7 +4,7 @@ * found in the LICENSE file. * * Helper functions related to fetching process information. - * Used by _psutil_osx module methods. + * Used by _psutil_macos module methods. */ @@ -142,7 +142,7 @@ psutil_get_cmdline(long pid) { mib[2] = (pid_t)pid; if (sysctl(mib, 3, procargs, &argmax, NULL, 0) < 0) { // In case of zombie process we'll get EINVAL. We translate it - // to NSP and _psosx.py will translate it to ZP. + // to NSP and _psmacos.py will translate it to ZP. if ((errno == EINVAL) && (psutil_pid_exists(pid))) NoSuchProcess(""); else @@ -236,7 +236,7 @@ psutil_get_environ(long pid) { mib[2] = (pid_t)pid; if (sysctl(mib, 3, procargs, &argmax, NULL, 0) < 0) { // In case of zombie process we'll get EINVAL. We translate it - // to NSP and _psosx.py will translate it to ZP. + // to NSP and _psmacos.py will translate it to ZP. if ((errno == EINVAL) && (psutil_pid_exists(pid))) NoSuchProcess(""); else diff --git a/psutil/arch/osx/process_info.h b/psutil/arch/macos/process_info.h similarity index 100% rename from psutil/arch/osx/process_info.h rename to psutil/arch/macos/process_info.h diff --git a/psutil/arch/osx/smc.c b/psutil/arch/macos/smc.c similarity index 100% rename from psutil/arch/osx/smc.c rename to psutil/arch/macos/smc.c diff --git a/psutil/arch/osx/smc.h b/psutil/arch/macos/smc.h similarity index 100% rename from psutil/arch/osx/smc.h rename to psutil/arch/macos/smc.h diff --git a/psutil/tests/README.rst b/psutil/tests/README.rst index 515abf772..9cefb7757 100644 --- a/psutil/tests/README.rst +++ b/psutil/tests/README.rst @@ -17,7 +17,7 @@ Instructions for running tests (``pip install tox``) then run ``tox`` from within psutil root directory. * Every time a commit is pushed tests are automatically run on Travis - (Linux, OSX) and appveyor (Windows): + (Linux, MACOS) and appveyor (Windows): * Travis builds: https://travis-ci.org/giampaolo/psutil * AppVeyor builds: https://ci.appveyor.com/project/giampaolo/psutil diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index f11a6a028..2b00d4263 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -36,7 +36,7 @@ from socket import SOCK_STREAM import psutil -from psutil import OSX +from psutil import MACOS from psutil import POSIX from psutil import SUNOS from psutil import WINDOWS @@ -179,7 +179,7 @@ def attempt(exe): else: return exe - if OSX: + if MACOS: exe = \ attempt(sys.executable) or \ attempt(os.path.realpath(sys.executable)) or \ @@ -367,7 +367,7 @@ def create_proc_children_pair(): def create_zombie_proc(): """Create a zombie process and return its PID.""" assert psutil.POSIX - unix_file = tempfile.mktemp(prefix=TESTFILE_PREFIX) if OSX else TESTFN + unix_file = tempfile.mktemp(prefix=TESTFILE_PREFIX) if MACOS else TESTFN src = textwrap.dedent("""\ import os, sys, time, socket, contextlib child_pid = os.fork() @@ -810,7 +810,7 @@ def get_suite(): x.startswith('test_memory_leaks')] if "WHEELHOUSE_UPLOADER_USERNAME" in os.environ: testmods = [x for x in testmods if not x.endswith(( - "osx", "posix", "linux"))] + "macos", "posix", "linux"))] suite = unittest.TestSuite() for tm in testmods: # ...so that the full test paths are printed on screen diff --git a/psutil/tests/test_connections.py b/psutil/tests/test_connections.py index 176e26648..cba835e14 100755 --- a/psutil/tests/test_connections.py +++ b/psutil/tests/test_connections.py @@ -18,9 +18,9 @@ import psutil from psutil import FREEBSD from psutil import LINUX +from psutil import MACOS from psutil import NETBSD from psutil import OPENBSD -from psutil import OSX from psutil import POSIX from psutil import SUNOS from psutil import WINDOWS @@ -124,9 +124,9 @@ def compare_procsys_connections(self, pid, proc_cons, kind='all'): try: sys_cons = psutil.net_connections(kind=kind) except psutil.AccessDenied: - # On OSX, system-wide connections are retrieved by iterating + # On MACOS, system-wide connections are retrieved by iterating # over all processes - if OSX: + if MACOS: return else: raise @@ -257,7 +257,7 @@ def test_unix(self): server.close() client.close() - @skip_on_access_denied(only_if=OSX) + @skip_on_access_denied(only_if=MACOS) def test_combos(self): def check_conn(proc, conn, family, type, laddr, raddr, status, kinds): all_kinds = ("all", "inet", "inet4", "inet6", "tcp", "tcp4", @@ -455,7 +455,7 @@ def test_multi_socks(self): @skip_on_access_denied() # See: https://travis-ci.org/giampaolo/psutil/jobs/237566297 - @unittest.skipIf(OSX and TRAVIS, "unreliable on OSX + TRAVIS") + @unittest.skipIf(MACOS and TRAVIS, "unreliable on MACOS + TRAVIS") def test_multi_sockets_procs(self): # Creates multiple sub processes, each creating different # sockets. For each process check that proc.connections() diff --git a/psutil/tests/test_contracts.py b/psutil/tests/test_contracts.py index 912e811f3..ac424b54f 100755 --- a/psutil/tests/test_contracts.py +++ b/psutil/tests/test_contracts.py @@ -21,9 +21,9 @@ from psutil import BSD from psutil import FREEBSD from psutil import LINUX +from psutil import MACOS from psutil import NETBSD from psutil import OPENBSD -from psutil import OSX from psutil import POSIX from psutil import SUNOS from psutil import WINDOWS @@ -113,21 +113,23 @@ def test_cpu_freq(self): linux = (LINUX and (os.path.exists("/sys/devices/system/cpu/cpufreq") or os.path.exists("/sys/devices/system/cpu/cpu0/cpufreq"))) - self.assertEqual(hasattr(psutil, "cpu_freq"), linux or OSX or WINDOWS) + self.assertEqual(hasattr(psutil, "cpu_freq"), + linux or MACOS or WINDOWS) def test_sensors_temperatures(self): - self.assertEqual(hasattr(psutil, "sensors_temperatures"), LINUX or OSX) + self.assertEqual( + hasattr(psutil, "sensors_temperatures"), LINUX or MACOS) def test_sensors_fans(self): - self.assertEqual(hasattr(psutil, "sensors_fans"), LINUX or OSX) + self.assertEqual(hasattr(psutil, "sensors_fans"), LINUX or MACOS) def test_battery(self): self.assertEqual(hasattr(psutil, "sensors_battery"), - LINUX or WINDOWS or FREEBSD or OSX) + LINUX or WINDOWS or FREEBSD or MACOS) def test_proc_environ(self): self.assertEqual(hasattr(psutil.Process, "environ"), - LINUX or OSX or WINDOWS) + LINUX or MACOS or WINDOWS) def test_proc_uids(self): self.assertEqual(hasattr(psutil.Process, "uids"), POSIX) @@ -146,7 +148,7 @@ def test_proc_rlimit(self): def test_proc_io_counters(self): hasit = hasattr(psutil.Process, "io_counters") - self.assertEqual(hasit, False if OSX or SUNOS else True) + self.assertEqual(hasit, False if MACOS or SUNOS else True) def test_proc_num_fds(self): self.assertEqual(hasattr(psutil.Process, "num_fds"), POSIX) @@ -224,7 +226,7 @@ def test_disk_partitions(self): @unittest.skipIf(not POSIX, 'POSIX only') @unittest.skipIf(not HAS_CONNECTIONS_UNIX, "can't list UNIX sockets") - @skip_on_access_denied(only_if=OSX) + @skip_on_access_denied(only_if=MACOS) def test_net_connections(self): with unix_socket_path() as name: with closing(bind_unix_socket(name)): @@ -385,7 +387,7 @@ def exe(self, ret, proc): # http://stackoverflow.com/questions/3112546/os-path-exists-lies if POSIX and os.path.isfile(ret): if hasattr(os, 'access') and hasattr(os, "X_OK"): - # XXX may fail on OSX + # XXX may fail on MACOS assert os.access(ret, os.X_OK) def pid(self, ret, proc): @@ -431,7 +433,7 @@ def gids(self, ret, proc): # gid == 30 (nodoby); not sure why. for gid in ret: self.assertIsInstance(gid, int) - if not OSX and not NETBSD: + if not MACOS and not NETBSD: self.assertGreaterEqual(gid, 0) self.assertIn(gid, self.all_gids) diff --git a/psutil/tests/test_osx.py b/psutil/tests/test_macos.py similarity index 97% rename from psutil/tests/test_osx.py rename to psutil/tests/test_macos.py index bcb2ba4e1..557af9f95 100755 --- a/psutil/tests/test_osx.py +++ b/psutil/tests/test_macos.py @@ -4,14 +4,14 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -"""OSX specific tests.""" +"""MACOS specific tests.""" import os import re import time import psutil -from psutil import OSX +from psutil import MACOS from psutil.tests import create_zombie_proc from psutil.tests import get_test_subprocess from psutil.tests import HAS_BATTERY @@ -23,7 +23,7 @@ from psutil.tests import unittest -PAGESIZE = os.sysconf("SC_PAGE_SIZE") if OSX else None +PAGESIZE = os.sysconf("SC_PAGE_SIZE") if MACOS else None def sysctl(cmdline): @@ -76,7 +76,7 @@ def human2bytes(s): return int(num * prefix[letter]) -@unittest.skipIf(not OSX, "OSX only") +@unittest.skipIf(not MACOS, "MACOS only") class TestProcess(unittest.TestCase): @classmethod @@ -101,7 +101,7 @@ def test_process_create_time(self): time.strftime("%Y", time.localtime(start_psutil))) -@unittest.skipIf(not OSX, "OSX only") +@unittest.skipIf(not MACOS, "MACOS only") class TestZombieProcessAPIs(unittest.TestCase): @classmethod @@ -162,7 +162,7 @@ def test_memory_maps(self): self.assertRaises(psutil.ZombieProcess, self.p.memory_maps) -@unittest.skipIf(not OSX, "OSX only") +@unittest.skipIf(not MACOS, "MACOS only") class TestSystemAPIs(unittest.TestCase): # --- disk diff --git a/psutil/tests/test_memory_leaks.py b/psutil/tests/test_memory_leaks.py index 680fe7803..ce0824596 100755 --- a/psutil/tests/test_memory_leaks.py +++ b/psutil/tests/test_memory_leaks.py @@ -25,8 +25,8 @@ import psutil import psutil._common from psutil import LINUX +from psutil import MACOS from psutil import OPENBSD -from psutil import OSX from psutil import POSIX from psutil import SUNOS from psutil import WINDOWS @@ -176,7 +176,7 @@ def call(): def _get_mem(): # By using USS memory it seems it's less likely to bump # into false positives. - if LINUX or WINDOWS or OSX: + if LINUX or WINDOWS or MACOS: return thisproc.memory_full_info().uss else: return thisproc.memory_info().rss @@ -344,8 +344,8 @@ def test_open_files(self): with open(TESTFN, 'w'): self.execute(self.proc.open_files) - # OSX implementation is unbelievably slow - @unittest.skipIf(OSX, "too slow on OSX") + # MACOS implementation is unbelievably slow + @unittest.skipIf(MACOS, "too slow on MACOS") @unittest.skipIf(not HAS_MEMORY_MAPS, "not supported") @skip_if_linux() def test_memory_maps(self): @@ -530,7 +530,7 @@ def test_net_io_counters(self): @unittest.skipIf(LINUX, "worthless on Linux (pure python)") - @unittest.skipIf(OSX and os.getuid() != 0, "need root access") + @unittest.skipIf(MACOS and os.getuid() != 0, "need root access") def test_net_connections(self): with create_sockets(): self.execute(psutil.net_connections) diff --git a/psutil/tests/test_posix.py b/psutil/tests/test_posix.py index e9a6f5f63..b80128c7f 100755 --- a/psutil/tests/test_posix.py +++ b/psutil/tests/test_posix.py @@ -19,8 +19,8 @@ from psutil import AIX from psutil import BSD from psutil import LINUX +from psutil import MACOS from psutil import OPENBSD -from psutil import OSX from psutil import POSIX from psutil import SUNOS from psutil._compat import PY3 @@ -152,7 +152,7 @@ def test_name(self): # remove path if there is any, from the command name_ps = os.path.basename(name_ps).lower() name_psutil = psutil.Process(self.pid).name().lower() - # ...because of how we calculate PYTHON_EXE; on OSX this may + # ...because of how we calculate PYTHON_EXE; on MACOS this may # be "pythonX.Y". name_ps = re.sub(r"\d.\d", "", name_ps) name_psutil = re.sub(r"\d.\d", "", name_psutil) @@ -194,7 +194,7 @@ def test_name_long_cmdline_nsp_exc(self): p = psutil.Process() self.assertRaises(psutil.NoSuchProcess, p.name) - @unittest.skipIf(OSX or BSD, 'ps -o start not available') + @unittest.skipIf(MACOS or BSD, 'ps -o start not available') def test_create_time(self): time_ps = ps("ps --no-headers -o start -p %s" % self.pid).split(' ')[0] time_psutil = psutil.Process(self.pid).create_time() @@ -309,8 +309,8 @@ def test_pids(self): pids_ps.sort() pids_psutil.sort() - # on OSX and OPENBSD ps doesn't show pid 0 - if OSX or OPENBSD and 0 not in pids_ps: + # on MACOS and OPENBSD ps doesn't show pid 0 + if MACOS or OPENBSD and 0 not in pids_ps: pids_ps.insert(0, 0) self.assertEqual(pids_ps, pids_psutil) diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index e295c95b1..230819623 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -25,9 +25,9 @@ from psutil import AIX from psutil import BSD from psutil import LINUX +from psutil import MACOS from psutil import NETBSD from psutil import OPENBSD -from psutil import OSX from psutil import POSIX from psutil import SUNOS from psutil import WINDOWS @@ -504,7 +504,7 @@ def test_rlimit_infinity_value(self): def test_num_threads(self): # on certain platforms such as Linux we might test for exact # thread number, since we always have with 1 thread per process, - # but this does not apply across all platforms (OSX, Windows) + # but this does not apply across all platforms (MACOS, Windows) p = psutil.Process() if OPENBSD: try: @@ -548,7 +548,7 @@ def test_threads(self): self.assertEqual(athread.system_time, athread[2]) @retry_before_failing() - @skip_on_access_denied(only_if=OSX) + @skip_on_access_denied(only_if=MACOS) @unittest.skipIf(not HAS_THREADS, 'not supported') def test_threads_2(self): sproc = get_test_subprocess() @@ -603,7 +603,7 @@ def test_memory_full_info(self): value = getattr(mem, name) self.assertGreaterEqual(value, 0, msg=(name, value)) self.assertLessEqual(value, total, msg=(name, value, total)) - if LINUX or WINDOWS or OSX: + if LINUX or WINDOWS or MACOS: self.assertGreaterEqual(mem.uss, 0) if LINUX: self.assertGreaterEqual(mem.pss, 0) @@ -668,7 +668,7 @@ def test_memory_percent(self): assert 0 <= ret <= 100, ret assert 0 <= ret <= 100, ret self.assertRaises(ValueError, p.memory_percent, memtype="?!?") - if LINUX or OSX or WINDOWS: + if LINUX or MACOS or WINDOWS: ret = p.memory_percent(memtype='uss') assert 0 <= ret <= 100, ret assert 0 <= ret <= 100, ret @@ -705,7 +705,7 @@ def test_exe(self): self.assertEqual(exe.replace(ver, ''), PYTHON_EXE.replace(ver, '')) except AssertionError: - # Tipically OSX. Really not sure what to do here. + # Tipically MACOS. Really not sure what to do here. pass out = sh([exe, "-c", "import os; print('hey')"]) @@ -825,8 +825,8 @@ def test_nice(self): self.assertEqual( os.getpriority(os.PRIO_PROCESS, os.getpid()), p.nice()) # XXX - going back to previous nice value raises - # AccessDenied on OSX - if not OSX: + # AccessDenied on MACOS + if not MACOS: p.nice(0) self.assertEqual(p.nice(), 0) except psutil.AccessDenied: @@ -1317,7 +1317,7 @@ def succeed_or_zombie_p_exc(fun, *args, **kwargs): succeed_or_zombie_p_exc(zproc.kill) # ...its parent should 'see' it - # edit: not true on BSD and OSX + # edit: not true on BSD and MACOS # descendants = [x.pid for x in psutil.Process().children( # recursive=True)] # self.assertIn(zpid, descendants) @@ -1327,9 +1327,9 @@ def succeed_or_zombie_p_exc(fun, *args, **kwargs): # self.assertEqual(zpid.ppid(), os.getpid()) # ...and all other APIs should be able to deal with it self.assertTrue(psutil.pid_exists(zpid)) - if not TRAVIS and OSX: + if not TRAVIS and MACOS: # For some reason this started failing all of the sudden. - # Maybe they upgraded OSX version? + # Maybe they upgraded MACOS version? # https://travis-ci.org/giampaolo/psutil/jobs/310896404 self.assertIn(zpid, psutil.pids()) self.assertIn(zpid, [x.pid for x in psutil.process_iter()]) @@ -1402,7 +1402,7 @@ def clean_dict(d): d.pop("PSUTIL_TESTING", None) d.pop("PLAT", None) d.pop("HOME", None) - if OSX: + if MACOS: d.pop("__CF_USER_TEXT_ENCODING", None) d.pop("VERSIONER_PYTHON_PREFER_32_BIT", None) d.pop("VERSIONER_PYTHON_VERSION", None) diff --git a/psutil/tests/test_system.py b/psutil/tests/test_system.py index 6081ea5a2..694a7a3b6 100755 --- a/psutil/tests/test_system.py +++ b/psutil/tests/test_system.py @@ -23,9 +23,9 @@ from psutil import BSD from psutil import FREEBSD from psutil import LINUX +from psutil import MACOS from psutil import NETBSD from psutil import OPENBSD -from psutil import OSX from psutil import POSIX from psutil import SUNOS from psutil import WINDOWS @@ -514,7 +514,7 @@ def test_disk_partitions(self): try: os.stat(disk.mountpoint) except OSError as err: - if TRAVIS and OSX and err.errno == errno.EIO: + if TRAVIS and MACOS and err.errno == errno.EIO: continue # http://mail.python.org/pipermail/python-dev/ # 2012-June/120787.html @@ -630,7 +630,7 @@ def test_net_if_addrs(self): elif addr.ptp: self.assertIsNone(addr.broadcast) - if BSD or OSX or SUNOS: + if BSD or MACOS or SUNOS: if hasattr(socket, "AF_LINK"): self.assertEqual(psutil.AF_LINK, socket.AF_LINK) elif LINUX: @@ -774,7 +774,7 @@ def check_ls(ls): self.assertEqual(len(ls), psutil.cpu_count()) def test_os_constants(self): - names = ["POSIX", "WINDOWS", "LINUX", "OSX", "FREEBSD", "OPENBSD", + names = ["POSIX", "WINDOWS", "LINUX", "MACOS", "FREEBSD", "OPENBSD", "NETBSD", "BSD", "SUNOS"] for name in names: self.assertIsInstance(getattr(psutil, name), bool, msg=name) @@ -799,8 +799,8 @@ def test_os_constants(self): assert psutil.SUNOS names.remove("SUNOS") elif "darwin" in sys.platform.lower(): - assert psutil.OSX - names.remove("OSX") + assert psutil.MACOS + names.remove("MACOS") else: assert psutil.WINDOWS assert not psutil.POSIX diff --git a/psutil/tests/test_unicode.py b/psutil/tests/test_unicode.py index 6383c9bec..4144b5c25 100755 --- a/psutil/tests/test_unicode.py +++ b/psutil/tests/test_unicode.py @@ -58,8 +58,8 @@ from contextlib import closing from psutil import BSD +from psutil import MACOS from psutil import OPENBSD -from psutil import OSX from psutil import POSIX from psutil import WINDOWS from psutil._compat import PY3 @@ -285,7 +285,7 @@ def normpath(p): self.assertIsInstance(path, str) -@unittest.skipIf(OSX and TRAVIS, "unreliable on TRAVIS") # TODO +@unittest.skipIf(MACOS and TRAVIS, "unreliable on TRAVIS") # TODO @unittest.skipIf(ASCII_FS, "ASCII fs") @unittest.skipIf(not subprocess_supports_unicode(TESTFN_UNICODE), "subprocess can't deal with unicode") @@ -306,7 +306,7 @@ def expect_exact_path_match(cls): return cls.funky_name in os.listdir(here) -@unittest.skipIf(OSX and TRAVIS, "unreliable on TRAVIS") # TODO +@unittest.skipIf(MACOS and TRAVIS, "unreliable on TRAVIS") # TODO @unittest.skipIf(not subprocess_supports_unicode(INVALID_NAME), "subprocess can't deal with invalid unicode") class TestFSAPIsWithInvalidPath(_BaseFSAPIsTests, unittest.TestCase): diff --git a/scripts/internal/bench_oneshot.py b/scripts/internal/bench_oneshot.py index 639e9ad76..28ad4bac8 100755 --- a/scripts/internal/bench_oneshot.py +++ b/scripts/internal/bench_oneshot.py @@ -79,7 +79,7 @@ 'terminal', 'uids', ] -elif psutil.OSX: +elif psutil.MACOS: names += [ 'cpu_times', 'create_time', diff --git a/scripts/internal/print_announce.py b/scripts/internal/print_announce.py index 018fb0928..c92cbcb2e 100755 --- a/scripts/internal/print_announce.py +++ b/scripts/internal/print_announce.py @@ -39,9 +39,9 @@ running processes. It implements many functionalities offered by command \ line tools such as: ps, top, lsof, netstat, ifconfig, who, df, kill, free, \ nice, ionice, iostat, iotop, uptime, pidof, tty, taskset, pmap. It \ -currently supports Linux, Windows, OSX, Sun Solaris, FreeBSD, OpenBSD, NetBSD \ -and AIX, both 32-bit and 64-bit architectures, with Python versions from 2.6 \ -to 3.6. PyPy is also known to work. +currently supports Linux, Windows, macOS, Sun Solaris, FreeBSD, OpenBSD, \ +NetBSD and AIX, both 32-bit and 64-bit architectures, with Python versions \ +from 2.6 to 3.6. PyPy is also known to work. What's new ========== diff --git a/scripts/iotop.py b/scripts/iotop.py index 9f76eb1c9..dabe957b5 100755 --- a/scripts/iotop.py +++ b/scripts/iotop.py @@ -8,7 +8,7 @@ A clone of iotop (http://guichaz.free.fr/iotop/) showing real time disk I/O statistics. -It works on Linux only (FreeBSD and OSX are missing support for IO +It works on Linux only (FreeBSD and macOS are missing support for IO counters). It doesn't work on Windows as curses module is required. diff --git a/scripts/pmap.py b/scripts/pmap.py index 16eebb609..a509bd733 100755 --- a/scripts/pmap.py +++ b/scripts/pmap.py @@ -5,7 +5,7 @@ # found in the LICENSE file. """ -A clone of 'pmap' utility on Linux, 'vmmap' on OSX and 'procstat -v' on BSD. +A clone of 'pmap' utility on Linux, 'vmmap' on macOS and 'procstat -v' on BSD. Report memory map of a process. $ python scripts/pmap.py 32402 diff --git a/scripts/procsmem.py b/scripts/procsmem.py index a28794b9d..ab9ad0666 100755 --- a/scripts/procsmem.py +++ b/scripts/procsmem.py @@ -41,7 +41,7 @@ import psutil -if not (psutil.LINUX or psutil.OSX or psutil.WINDOWS): +if not (psutil.LINUX or psutil.MACOS or psutil.WINDOWS): sys.exit("platform not supported") diff --git a/setup.py b/setup.py index b0343212b..03e315b54 100755 --- a/setup.py +++ b/setup.py @@ -28,16 +28,16 @@ # ...so we can import _common.py sys.path.insert(0, os.path.join(HERE, "psutil")) +from _common import AIX # NOQA from _common import BSD # NOQA from _common import FREEBSD # NOQA from _common import LINUX # NOQA +from _common import MACOS # NOQA from _common import NETBSD # NOQA from _common import OPENBSD # NOQA -from _common import OSX # NOQA from _common import POSIX # NOQA from _common import SUNOS # NOQA from _common import WINDOWS # NOQA -from _common import AIX # NOQA macros = [] @@ -145,14 +145,14 @@ def get_winver(): # extra_link_args=["/DEBUG"] ) -elif OSX: - macros.append(("PSUTIL_OSX", 1)) +elif MACOS: + macros.append(("PSUTIL_MACOS", 1)) ext = Extension( - 'psutil._psutil_osx', + 'psutil._psutil_macos', sources=sources + [ - 'psutil/_psutil_osx.c', - 'psutil/arch/osx/process_info.c', - 'psutil/arch/osx/smc.c', + 'psutil/_psutil_macos.c', + 'psutil/arch/macos/process_info.c', + 'psutil/arch/macos/smc.c', ], define_macros=macros, extra_link_args=[ From 05066d9744f0b981f612c71eecace817a2ba7a08 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 29 Jun 2018 00:50:08 +0200 Subject: [PATCH 090/182] winmake: add upload-wheels cmd --- docs/index.rst | 2 +- scripts/internal/winmake.py | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index 47167d764..af207702a 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -32,7 +32,7 @@ psutil currently supports the following platforms: - **Linux** - **Windows** -- **macOS**, +- **macOS** - **FreeBSD, OpenBSD**, **NetBSD** - **Sun Solaris** - **AIX** diff --git a/scripts/internal/winmake.py b/scripts/internal/winmake.py index ff3a64891..ffdfc291e 100755 --- a/scripts/internal/winmake.py +++ b/scripts/internal/winmake.py @@ -227,6 +227,13 @@ def wheel(): sh("%s setup.py bdist_wheel" % PYTHON) +@cmd +def upload_wheels(): + """Upload wheel files on PYPI.""" + build() + sh("%s -m twine upload dist/*.whl" % PYTHON) + + @cmd def install_pip(): """Install pip""" From 1b09b5fff78f705dfb42458726ff9789c26f6f21 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 29 Jun 2018 16:37:47 +0200 Subject: [PATCH 091/182] revert file renaming of macos files; get them back to 'osx' prefix --- MANIFEST.in | 14 +++++------ Makefile | 2 +- psutil/__init__.py | 2 +- psutil/{_psmacos.py => _psosx.py} | 2 +- psutil/{_psutil_macos.c => _psutil_osx.c} | 26 ++++++++++----------- psutil/_psutil_posix.c | 16 ++++++------- psutil/arch/{macos => osx}/process_info.c | 6 ++--- psutil/arch/{macos => osx}/process_info.h | 0 psutil/arch/{macos => osx}/smc.c | 0 psutil/arch/{macos => osx}/smc.h | 0 psutil/tests/__init__.py | 2 +- psutil/tests/{test_macos.py => test_osx.py} | 0 setup.py | 10 ++++---- 13 files changed, 40 insertions(+), 40 deletions(-) rename psutil/{_psmacos.py => _psosx.py} (99%) rename psutil/{_psutil_macos.c => _psutil_osx.c} (99%) rename psutil/arch/{macos => osx}/process_info.c (98%) rename psutil/arch/{macos => osx}/process_info.h (100%) rename psutil/arch/{macos => osx}/smc.c (100%) rename psutil/arch/{macos => osx}/smc.h (100%) rename psutil/tests/{test_macos.py => test_osx.py} (100%) diff --git a/MANIFEST.in b/MANIFEST.in index c6d8c550a..146cd92df 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -28,7 +28,7 @@ include psutil/_exceptions.py include psutil/_psaix.py include psutil/_psbsd.py include psutil/_pslinux.py -include psutil/_psmacos.py +include psutil/_psosx.py include psutil/_psposix.py include psutil/_pssunos.py include psutil/_psutil_aix.c @@ -36,7 +36,7 @@ include psutil/_psutil_bsd.c include psutil/_psutil_common.c include psutil/_psutil_common.h include psutil/_psutil_linux.c -include psutil/_psutil_macos.c +include psutil/_psutil_osx.c include psutil/_psutil_posix.c include psutil/_psutil_posix.h include psutil/_psutil_sunos.c @@ -55,16 +55,16 @@ include psutil/arch/freebsd/specific.c include psutil/arch/freebsd/specific.h include psutil/arch/freebsd/sys_socks.c include psutil/arch/freebsd/sys_socks.h -include psutil/arch/macos/process_info.c -include psutil/arch/macos/process_info.h -include psutil/arch/macos/smc.c -include psutil/arch/macos/smc.h include psutil/arch/netbsd/socks.c include psutil/arch/netbsd/socks.h include psutil/arch/netbsd/specific.c include psutil/arch/netbsd/specific.h include psutil/arch/openbsd/specific.c include psutil/arch/openbsd/specific.h +include psutil/arch/osx/process_info.c +include psutil/arch/osx/process_info.h +include psutil/arch/osx/smc.c +include psutil/arch/osx/smc.h include psutil/arch/solaris/environ.c include psutil/arch/solaris/environ.h include psutil/arch/solaris/v10/ifaddrs.c @@ -88,7 +88,7 @@ include psutil/tests/test_bsd.py include psutil/tests/test_connections.py include psutil/tests/test_contracts.py include psutil/tests/test_linux.py -include psutil/tests/test_macos.py +include psutil/tests/test_osx.py include psutil/tests/test_memory_leaks.py include psutil/tests/test_misc.py include psutil/tests/test_posix.py diff --git a/Makefile b/Makefile index 58c0d3961..51047383f 100644 --- a/Makefile +++ b/Makefile @@ -142,7 +142,7 @@ test-posix: ## POSIX specific tests. test-platform: ## Run specific platform tests only. ${MAKE} install - $(TEST_PREFIX) $(PYTHON) psutil/tests/test_`$(PYTHON) -c 'import psutil; print([x.lower() for x in ("LINUX", "BSD", "MACOS", "SUNOS", "WINDOWS", "AIX") if getattr(psutil, x)][0])'`.py + $(TEST_PREFIX) $(PYTHON) psutil/tests/test_`$(PYTHON) -c 'import psutil; print([x.lower() for x in ("LINUX", "BSD", "OSX", "SUNOS", "WINDOWS", "AIX") if getattr(psutil, x)][0])'`.py test-memleaks: ## Memory leak tests. ${MAKE} install diff --git a/psutil/__init__.py b/psutil/__init__.py index affc5c8c3..43e9aa7e9 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -153,7 +153,7 @@ from ._pswindows import CONN_DELETE_TCB # NOQA elif MACOS: - from . import _psmacos as _psplatform + from . import _psosx as _psplatform elif BSD: from . import _psbsd as _psplatform diff --git a/psutil/_psmacos.py b/psutil/_psosx.py similarity index 99% rename from psutil/_psmacos.py rename to psutil/_psosx.py index a719bedaa..6c77dc691 100644 --- a/psutil/_psmacos.py +++ b/psutil/_psosx.py @@ -14,7 +14,7 @@ from . import _common from . import _psposix -from . import _psutil_macos as cext +from . import _psutil_osx as cext from . import _psutil_posix as cext_posix from ._common import AF_INET6 from ._common import conn_tmap diff --git a/psutil/_psutil_macos.c b/psutil/_psutil_osx.c similarity index 99% rename from psutil/_psutil_macos.c rename to psutil/_psutil_osx.c index 8fc3140f6..e3250caba 100644 --- a/psutil/_psutil_macos.c +++ b/psutil/_psutil_osx.c @@ -43,8 +43,8 @@ #include "_psutil_common.h" #include "_psutil_posix.h" -#include "arch/macos/process_info.h" -#include "arch/macos/smc.h" +#include "arch/osx/process_info.h" +#include "arch/osx/smc.h" #define PSUTIL_TV2DOUBLE(t) ((t).tv_sec + (t).tv_usec / 1000000.0) @@ -342,7 +342,7 @@ psutil_proc_cmdline(PyObject *self, PyObject *args) { if (! PyArg_ParseTuple(args, "l", &pid)) return NULL; - // get the commandline, defined in arch/macos/process_info.c + // get the commandline, defined in arch/osx/process_info.c py_retlist = psutil_get_cmdline(pid); return py_retlist; } @@ -359,7 +359,7 @@ psutil_proc_environ(PyObject *self, PyObject *args) { if (! PyArg_ParseTuple(args, "l", &pid)) return NULL; - // get the environment block, defined in arch/macos/process_info.c + // get the environment block, defined in arch/osx/process_info.c py_retdict = psutil_get_environ(pid); return py_retdict; } @@ -2066,13 +2066,13 @@ struct module_state { #if PY_MAJOR_VERSION >= 3 static int -psutil_macos_traverse(PyObject *m, visitproc visit, void *arg) { +psutil_osx_traverse(PyObject *m, visitproc visit, void *arg) { Py_VISIT(GETSTATE(m)->error); return 0; } static int -psutil_macos_clear(PyObject *m) { +psutil_osx_clear(PyObject *m) { Py_CLEAR(GETSTATE(m)->error); return 0; } @@ -2080,31 +2080,31 @@ psutil_macos_clear(PyObject *m) { static struct PyModuleDef moduledef = { PyModuleDef_HEAD_INIT, - "psutil_macos", + "psutil_osx", NULL, sizeof(struct module_state), PsutilMethods, NULL, - psutil_macos_traverse, - psutil_macos_clear, + psutil_osx_traverse, + psutil_osx_clear, NULL }; #define INITERROR return NULL -PyMODINIT_FUNC PyInit__psutil_macos(void) +PyMODINIT_FUNC PyInit__psutil_osx(void) #else #define INITERROR return void -init_psutil_macos(void) +init_psutil_osx(void) #endif { #if PY_MAJOR_VERSION >= 3 PyObject *module = PyModule_Create(&moduledef); #else - PyObject *module = Py_InitModule("_psutil_macos", PsutilMethods); + PyObject *module = Py_InitModule("_psutil_osx", PsutilMethods); #endif PyModule_AddIntConstant(module, "version", PSUTIL_VERSION); // process status constants, defined in: @@ -2130,7 +2130,7 @@ init_psutil_macos(void) // Exception. ZombieProcessError = PyErr_NewException( - "_psutil_macos.ZombieProcessError", NULL, NULL); + "_psutil_osx.ZombieProcessError", NULL, NULL); Py_INCREF(ZombieProcessError); PyModule_AddObject(module, "ZombieProcessError", ZombieProcessError); diff --git a/psutil/_psutil_posix.c b/psutil/_psutil_posix.c index e9a8f2d95..c851abbc1 100644 --- a/psutil/_psutil_posix.c +++ b/psutil/_psutil_posix.c @@ -28,7 +28,7 @@ #include #include #include -#elif defined(PSUTIL_BSD) || defined(PSUTIL_MACOS) +#elif defined(PSUTIL_BSD) || defined(PSUTIL_OSX) #include #include #include @@ -71,7 +71,7 @@ psutil_pid_exists(long pid) { #endif } -#if defined(PSUTIL_MACOS) +#if defined(PSUTIL_OSX) ret = kill((pid_t)pid , 0); #else ret = kill(pid , 0); @@ -143,7 +143,7 @@ psutil_posix_getpriority(PyObject *self, PyObject *args) { if (! PyArg_ParseTuple(args, "l", &pid)) return NULL; -#ifdef PSUTIL_MACOS +#ifdef PSUTIL_OSX priority = getpriority(PRIO_PROCESS, (id_t)pid); #else priority = getpriority(PRIO_PROCESS, pid); @@ -166,7 +166,7 @@ psutil_posix_setpriority(PyObject *self, PyObject *args) { if (! PyArg_ParseTuple(args, "li", &pid, &priority)) return NULL; -#ifdef PSUTIL_MACOS +#ifdef PSUTIL_OSX retval = setpriority(PRIO_PROCESS, (id_t)pid, priority); #else retval = setpriority(PRIO_PROCESS, pid, priority); @@ -221,7 +221,7 @@ psutil_convert_ipaddr(struct sockaddr *addr, int family) { len = lladdr->sll_halen; data = (const char *)lladdr->sll_addr; } -#elif defined(PSUTIL_BSD) || defined(PSUTIL_MACOS) +#elif defined(PSUTIL_BSD) || defined(PSUTIL_OSX) else if (addr->sa_family == AF_LINK) { // Note: prior to Python 3.4 socket module does not expose // AF_LINK so we'll do. @@ -433,7 +433,7 @@ psutil_net_if_flags(PyObject *self, PyObject *args) { /* * net_if_stats() macOS/BSD implementation. */ -#if defined(PSUTIL_BSD) || defined(PSUTIL_MACOS) +#if defined(PSUTIL_BSD) || defined(PSUTIL_OSX) int psutil_get_nic_speed(int ifm_active) { // Determine NIC speed. Taken from: @@ -638,7 +638,7 @@ PsutilMethods[] = { "Retrieve NIC MTU"}, {"net_if_flags", psutil_net_if_flags, METH_VARARGS, "Retrieve NIC flags"}, -#if defined(PSUTIL_BSD) || defined(PSUTIL_MACOS) +#if defined(PSUTIL_BSD) || defined(PSUTIL_OSX) {"net_if_duplex_speed", psutil_net_if_duplex_speed, METH_VARARGS, "Return NIC stats."}, #endif @@ -698,7 +698,7 @@ void init_psutil_posix(void) PyObject *module = Py_InitModule("_psutil_posix", PsutilMethods); #endif -#if defined(PSUTIL_BSD) || defined(PSUTIL_MACOS) || defined(PSUTIL_SUNOS) || defined(PSUTIL_AIX) +#if defined(PSUTIL_BSD) || defined(PSUTIL_OSX) || defined(PSUTIL_SUNOS) || defined(PSUTIL_AIX) PyModule_AddIntConstant(module, "AF_LINK", AF_LINK); #endif diff --git a/psutil/arch/macos/process_info.c b/psutil/arch/osx/process_info.c similarity index 98% rename from psutil/arch/macos/process_info.c rename to psutil/arch/osx/process_info.c index de007de48..40c79a2cd 100644 --- a/psutil/arch/macos/process_info.c +++ b/psutil/arch/osx/process_info.c @@ -4,7 +4,7 @@ * found in the LICENSE file. * * Helper functions related to fetching process information. - * Used by _psutil_macos module methods. + * Used by _psutil_osx module methods. */ @@ -142,7 +142,7 @@ psutil_get_cmdline(long pid) { mib[2] = (pid_t)pid; if (sysctl(mib, 3, procargs, &argmax, NULL, 0) < 0) { // In case of zombie process we'll get EINVAL. We translate it - // to NSP and _psmacos.py will translate it to ZP. + // to NSP and _psosx.py will translate it to ZP. if ((errno == EINVAL) && (psutil_pid_exists(pid))) NoSuchProcess(""); else @@ -236,7 +236,7 @@ psutil_get_environ(long pid) { mib[2] = (pid_t)pid; if (sysctl(mib, 3, procargs, &argmax, NULL, 0) < 0) { // In case of zombie process we'll get EINVAL. We translate it - // to NSP and _psmacos.py will translate it to ZP. + // to NSP and _psosx.py will translate it to ZP. if ((errno == EINVAL) && (psutil_pid_exists(pid))) NoSuchProcess(""); else diff --git a/psutil/arch/macos/process_info.h b/psutil/arch/osx/process_info.h similarity index 100% rename from psutil/arch/macos/process_info.h rename to psutil/arch/osx/process_info.h diff --git a/psutil/arch/macos/smc.c b/psutil/arch/osx/smc.c similarity index 100% rename from psutil/arch/macos/smc.c rename to psutil/arch/osx/smc.c diff --git a/psutil/arch/macos/smc.h b/psutil/arch/osx/smc.h similarity index 100% rename from psutil/arch/macos/smc.h rename to psutil/arch/osx/smc.h diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 2b00d4263..d293498e8 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -810,7 +810,7 @@ def get_suite(): x.startswith('test_memory_leaks')] if "WHEELHOUSE_UPLOADER_USERNAME" in os.environ: testmods = [x for x in testmods if not x.endswith(( - "macos", "posix", "linux"))] + "osx", "posix", "linux"))] suite = unittest.TestSuite() for tm in testmods: # ...so that the full test paths are printed on screen diff --git a/psutil/tests/test_macos.py b/psutil/tests/test_osx.py similarity index 100% rename from psutil/tests/test_macos.py rename to psutil/tests/test_osx.py diff --git a/setup.py b/setup.py index 03e315b54..adc7a0e2d 100755 --- a/setup.py +++ b/setup.py @@ -146,13 +146,13 @@ def get_winver(): ) elif MACOS: - macros.append(("PSUTIL_MACOS", 1)) + macros.append(("PSUTIL_OSX", 1)) ext = Extension( - 'psutil._psutil_macos', + 'psutil._psutil_osx', sources=sources + [ - 'psutil/_psutil_macos.c', - 'psutil/arch/macos/process_info.c', - 'psutil/arch/macos/smc.c', + 'psutil/_psutil_osx.c', + 'psutil/arch/osx/process_info.c', + 'psutil/arch/osx/smc.c', ], define_macros=macros, extra_link_args=[ From f440b6e3c6e9bbcd3e57f9c0d2f78aef36a249b1 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 15 Jul 2018 15:16:47 +0200 Subject: [PATCH 092/182] appveyor: retire 3.5, add 3.7 --- appveyor.yml | 25 +++++++------------------ scripts/internal/download_exes.py | 2 +- 2 files changed, 8 insertions(+), 19 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 436faadbc..ee434903a 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -3,27 +3,18 @@ os: Visual Studio 2015 environment: - global: # SDK v7.0 MSVC Express 2008's SetEnv.cmd script will fail if the # /E:ON and /V:ON options are not enabled in the batch script intepreter # See: http://stackoverflow.com/a/13751649/163740 WITH_COMPILER: "cmd /E:ON /V:ON /C .\\.ci\\appveyor\\run_with_compiler.cmd" - matrix: - # Pre-installed Python versions, which Appveyor may upgrade to - # a later point release. - # 32 bits - PYTHON: "C:\\Python27" PYTHON_VERSION: "2.7.x" PYTHON_ARCH: "32" - - PYTHON: "C:\\Python34" - PYTHON_VERSION: "3.4.x" - PYTHON_ARCH: "32" - - PYTHON: "C:\\Python35" PYTHON_VERSION: "3.5.x" PYTHON_ARCH: "32" @@ -32,16 +23,16 @@ environment: PYTHON_VERSION: "3.6.x" PYTHON_ARCH: "32" + - PYTHON: "C:\\Python37" + PYTHON_VERSION: "3.7.x" + PYTHON_ARCH: "32" + # 64 bits - PYTHON: "C:\\Python27-x64" PYTHON_VERSION: "2.7.x" PYTHON_ARCH: "64" - - PYTHON: "C:\\Python34-x64" - PYTHON_VERSION: "3.4.x" - PYTHON_ARCH: "64" - - PYTHON: "C:\\Python35-x64" PYTHON_VERSION: "3.5.x" PYTHON_ARCH: "64" @@ -50,11 +41,9 @@ environment: PYTHON_VERSION: "3.6.x" PYTHON_ARCH: "64" - # Also build on a Python version not pre-installed by Appveyor. - # See: https://github.com/ogrisel/python-appveyor-demo/issues/10 - # - PYTHON: "C:\\Python266" - # PYTHON_VERSION: "2.6.6" - # PYTHON_ARCH: "32" + - PYTHON: "C:\\Python37-x64" + PYTHON_VERSION: "3.7.x" + PYTHON_ARCH: "64" init: - "ECHO %PYTHON% %PYTHON_VERSION% %PYTHON_ARCH%" diff --git a/scripts/internal/download_exes.py b/scripts/internal/download_exes.py index 1b0044288..d138f0d36 100755 --- a/scripts/internal/download_exes.py +++ b/scripts/internal/download_exes.py @@ -25,7 +25,7 @@ BASE_URL = 'https://ci.appveyor.com/api' -PY_VERSIONS = ['2.7', '3.4', '3.5', '3.6'] +PY_VERSIONS = ['2.7', '3.5', '3.6', '3.7'] TIMEOUT = 30 COLORS = True From cf9d1cd9045e4d73ecd2d7308183d254081bc9b3 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 16 Jul 2018 00:10:03 +0200 Subject: [PATCH 093/182] fix #1279: catch and skip ENODEV in net_if_stat() --- HISTORY.rst | 1 + docs/index.rst | 8 ++++---- psutil/_psbsd.py | 18 ++++++++++++------ psutil/_pslinux.py | 14 ++++++++++---- psutil/_psosx.py | 18 ++++++++++++------ psutil/tests/test_system.py | 10 ++++++++++ 6 files changed, 49 insertions(+), 20 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 67f967eda..a5382fbcf 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -18,6 +18,7 @@ XXXX-XX-XX task_for_pid() syscall. AccessDenied is now raised instead. - 1278_: [macOS] Process.threads() incorrectly return microseconds instead of seconds. (patch by Nikhil Marathe) +- 1279_: [Linux, macOS, BSD] net_if_stats() may return ENODEV. 5.4.6 ===== diff --git a/docs/index.rst b/docs/index.rst index af207702a..d17d2c657 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1880,11 +1880,11 @@ Process class (OpenBSD) "laddr" and "raddr" fields for UNIX sockets are always set to "". This is a limitation of the OS. - .. versionchanged:: 5.3.0 : "laddr" and "raddr" are named tuples. + .. note:: + (AIX) :class:`psutil.AccessDenied` is always raised unless running + as root (lsof does the same). - .. note:: - (AIX) :class:`psutil.AccessDenied` is always raised unless running - as root (lsof does the same). + .. versionchanged:: 5.3.0 : "laddr" and "raddr" are named tuples. .. method:: is_running() diff --git a/psutil/_psbsd.py b/psutil/_psbsd.py index 83f38d55e..7f4bcb6de 100644 --- a/psutil/_psbsd.py +++ b/psutil/_psbsd.py @@ -345,12 +345,18 @@ def net_if_stats(): names = net_io_counters().keys() ret = {} for name in names: - mtu = cext_posix.net_if_mtu(name) - isup = cext_posix.net_if_flags(name) - duplex, speed = cext_posix.net_if_duplex_speed(name) - if hasattr(_common, 'NicDuplex'): - duplex = _common.NicDuplex(duplex) - ret[name] = _common.snicstats(isup, duplex, speed, mtu) + try: + mtu = cext_posix.net_if_mtu(name) + isup = cext_posix.net_if_flags(name) + duplex, speed = cext_posix.net_if_duplex_speed(name) + except OSError as err: + # https://github.com/giampaolo/psutil/issues/1279 + if err.errno != errno.ENODEV: + raise + else: + if hasattr(_common, 'NicDuplex'): + duplex = _common.NicDuplex(duplex) + ret[name] = _common.snicstats(isup, duplex, speed, mtu) return ret diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index b197dba33..85ffe66a5 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -1002,10 +1002,16 @@ def net_if_stats(): names = net_io_counters().keys() ret = {} for name in names: - mtu = cext_posix.net_if_mtu(name) - isup = cext_posix.net_if_flags(name) - duplex, speed = cext.net_if_duplex_speed(name) - ret[name] = _common.snicstats(isup, duplex_map[duplex], speed, mtu) + try: + mtu = cext_posix.net_if_mtu(name) + isup = cext_posix.net_if_flags(name) + duplex, speed = cext.net_if_duplex_speed(name) + except OSError as err: + # https://github.com/giampaolo/psutil/issues/1279 + if err.errno != errno.ENODEV: + raise + else: + ret[name] = _common.snicstats(isup, duplex_map[duplex], speed, mtu) return ret diff --git a/psutil/_psosx.py b/psutil/_psosx.py index 6c77dc691..d059449af 100644 --- a/psutil/_psosx.py +++ b/psutil/_psosx.py @@ -402,12 +402,18 @@ def net_if_stats(): names = net_io_counters().keys() ret = {} for name in names: - mtu = cext_posix.net_if_mtu(name) - isup = cext_posix.net_if_flags(name) - duplex, speed = cext_posix.net_if_duplex_speed(name) - if hasattr(_common, 'NicDuplex'): - duplex = _common.NicDuplex(duplex) - ret[name] = _common.snicstats(isup, duplex, speed, mtu) + try: + mtu = cext_posix.net_if_mtu(name) + isup = cext_posix.net_if_flags(name) + duplex, speed = cext_posix.net_if_duplex_speed(name) + except OSError as err: + # https://github.com/giampaolo/psutil/issues/1279 + if err.errno != errno.ENODEV: + raise + else: + if hasattr(_common, 'NicDuplex'): + duplex = _common.NicDuplex(duplex) + ret[name] = _common.snicstats(isup, duplex, speed, mtu) return ret diff --git a/psutil/tests/test_system.py b/psutil/tests/test_system.py index 694a7a3b6..18cbb5d8b 100755 --- a/psutil/tests/test_system.py +++ b/psutil/tests/test_system.py @@ -671,6 +671,16 @@ def test_net_if_stats(self): self.assertGreaterEqual(speed, 0) self.assertGreaterEqual(mtu, 0) + @unittest.skipIf(not (LINUX or BSD or MACOS), + "LINUX or BSD or MACOS specific") + def test_net_if_stats_enodev(self): + # See: https://github.com/giampaolo/psutil/issues/1279 + with mock.patch('psutil._psutil_posix.net_if_mtu', + side_effect=OSError(errno.ENODEV, "")) as m: + ret = psutil.net_if_stats() + self.assertEqual(ret, {}) + assert m.called + @unittest.skipIf(LINUX and not os.path.exists('/proc/diskstats'), '/proc/diskstats not available on this linux version') @unittest.skipIf(APPVEYOR and psutil.disk_io_counters() is None, From 07b42c153d3d2ad11d49e8876a2128ac00f0cffb Mon Sep 17 00:00:00 2001 From: Ashish Billore Date: Tue, 24 Jul 2018 17:39:19 +0900 Subject: [PATCH 094/182] Update index.rst (#1308) Updated Process section --- docs/index.rst | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index d17d2c657..35f135683 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -848,10 +848,8 @@ Functions Cached :class:`Process` instances are checked for identity so that you're safe in case a PID has been reused by another process, in which case the cached instance is updated. - This is should be preferred over :func:`psutil.pids()` for iterating over - processes. - Sorting order in which processes are returned is - based on their PID. + This is preferred over :func:`psutil.pids()` for iterating over processes. + Sorting order in which processes are returned is based on their PID. *attrs* and *ad_value* have the same meaning as in :meth:`Process.as_dict()`. If *attrs* is specified :meth:`Process.as_dict()` is called internally and the resulting dict is stored as a ``info`` attribute which is attached to the From e2d94439387d176f5f84144649e661b0cfcb5969 Mon Sep 17 00:00:00 2001 From: sylvainduchesne <35115147+sylvainduchesne@users.noreply.github.com> Date: Tue, 24 Jul 2018 16:09:39 -0400 Subject: [PATCH 095/182] retain GIL when querying connections table (#1306) fixes issue #1294 --- psutil/_psutil_windows.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/psutil/_psutil_windows.c b/psutil/_psutil_windows.c index ba898f60b..1d44dd0ba 100644 --- a/psutil/_psutil_windows.c +++ b/psutil/_psutil_windows.c @@ -1536,7 +1536,6 @@ static DWORD __GetExtendedTcpTable(_GetExtendedTcpTable call, DWORD error = ERROR_INSUFFICIENT_BUFFER; *size = 0; *data = NULL; - Py_BEGIN_ALLOW_THREADS; error = call(NULL, size, FALSE, address_family, TCP_TABLE_OWNER_PID_ALL, 0); while (error == ERROR_INSUFFICIENT_BUFFER) @@ -1553,7 +1552,6 @@ static DWORD __GetExtendedTcpTable(_GetExtendedTcpTable call, *data = NULL; } } - Py_END_ALLOW_THREADS; return error; } @@ -1577,7 +1575,6 @@ static DWORD __GetExtendedUdpTable(_GetExtendedUdpTable call, DWORD error = ERROR_INSUFFICIENT_BUFFER; *size = 0; *data = NULL; - Py_BEGIN_ALLOW_THREADS; error = call(NULL, size, FALSE, address_family, UDP_TABLE_OWNER_PID, 0); while (error == ERROR_INSUFFICIENT_BUFFER) @@ -1594,7 +1591,6 @@ static DWORD __GetExtendedUdpTable(_GetExtendedUdpTable call, *data = NULL; } } - Py_END_ALLOW_THREADS; return error; } From 35cb1ab6e7cf2ef0942a135d812eebdce5af958d Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 24 Jul 2018 22:11:48 +0200 Subject: [PATCH 096/182] give CREDITS to @sylvainduchesne for #1294 --- CREDITS | 4 ++++ HISTORY.rst | 2 ++ 2 files changed, 6 insertions(+) diff --git a/CREDITS b/CREDITS index 89a91008e..edf1799dd 100644 --- a/CREDITS +++ b/CREDITS @@ -547,3 +547,7 @@ I: 1278 N: Alex Manuskin W: https://github.com/amanusk I: 1284 + +N: sylvainduchesne +W: https://github.com/sylvainduchesne +I: 1294 diff --git a/HISTORY.rst b/HISTORY.rst index a5382fbcf..2a39e1c61 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -19,6 +19,8 @@ XXXX-XX-XX - 1278_: [macOS] Process.threads() incorrectly return microseconds instead of seconds. (patch by Nikhil Marathe) - 1279_: [Linux, macOS, BSD] net_if_stats() may return ENODEV. +- 1294_: [Windows] psutil.Process().connections() may sometime fail with + MemoryError. (patch by sylvainduchesne) 5.4.6 ===== From df46de5d4df8da20f6939f121a151a646cf88ed2 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 25 Jul 2018 20:54:17 +0200 Subject: [PATCH 097/182] fix #1309: add STATUS_PARKED constant and fix STATUS_IDLE (both on linux) --- HISTORY.rst | 3 +++ docs/index.rst | 4 +++- psutil/__init__.py | 6 ++++-- psutil/_common.py | 5 +++-- psutil/_pslinux.py | 9 +++++++-- 5 files changed, 20 insertions(+), 7 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 2a39e1c61..602ce3d76 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -11,6 +11,7 @@ XXXX-XX-XX (patch by Alex Manuskin) - 1286_: [macOS] psutil.OSX constant is now deprecated in favor of new psutil.MACOS. +- 1309_: [Linux] added psutil.STATUS_PARKED constant for Process.status(). **Bug fixes** @@ -21,6 +22,8 @@ XXXX-XX-XX - 1279_: [Linux, macOS, BSD] net_if_stats() may return ENODEV. - 1294_: [Windows] psutil.Process().connections() may sometime fail with MemoryError. (patch by sylvainduchesne) +- 1309_: [Linux] Process.status() is unable to recognie "idle" and "parked" + statuses (returns '?'). 5.4.6 ===== diff --git a/docs/index.rst b/docs/index.rst index 35f135683..41b7175f9 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -2169,7 +2169,8 @@ Constants .. data:: STATUS_DEAD .. data:: STATUS_WAKE_KILL .. data:: STATUS_WAKING -.. data:: STATUS_IDLE (macOS, FreeBSD) +.. data:: STATUS_PARKED (Linux) +.. data:: STATUS_IDLE (Linux, macOS, FreeBSD) .. data:: STATUS_LOCKED (FreeBSD) .. data:: STATUS_WAITING (FreeBSD) .. data:: STATUS_SUSPENDED (NetBSD) @@ -2178,6 +2179,7 @@ Constants Returned by :meth:`psutil.Process.status()`. .. versionadded:: 3.4.1 STATUS_SUSPENDED (NetBSD) + .. versionadded:: 5.5.0 STATUS_PARKED (Linux) .. _const-conn: .. data:: CONN_ESTABLISHED diff --git a/psutil/__init__.py b/psutil/__init__.py index 43e9aa7e9..299fc1ea8 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -48,13 +48,14 @@ from ._common import STATUS_DEAD from ._common import STATUS_DISK_SLEEP -from ._common import STATUS_IDLE # bsd +from ._common import STATUS_IDLE from ._common import STATUS_LOCKED +from ._common import STATUS_PARKED from ._common import STATUS_RUNNING from ._common import STATUS_SLEEPING from ._common import STATUS_STOPPED from ._common import STATUS_TRACING_STOP -from ._common import STATUS_WAITING # bsd +from ._common import STATUS_WAITING from ._common import STATUS_WAKING from ._common import STATUS_ZOMBIE @@ -189,6 +190,7 @@ "STATUS_RUNNING", "STATUS_IDLE", "STATUS_SLEEPING", "STATUS_DISK_SLEEP", "STATUS_STOPPED", "STATUS_TRACING_STOP", "STATUS_ZOMBIE", "STATUS_DEAD", "STATUS_WAKING", "STATUS_LOCKED", "STATUS_WAITING", "STATUS_LOCKED", + "STATUS_PARKED", "CONN_ESTABLISHED", "CONN_SYN_SENT", "CONN_SYN_RECV", "CONN_FIN_WAIT1", "CONN_FIN_WAIT2", "CONN_TIME_WAIT", "CONN_CLOSE", "CONN_CLOSE_WAIT", diff --git a/psutil/_common.py b/psutil/_common.py index 6fcf8fe05..2cc3939a1 100644 --- a/psutil/_common.py +++ b/psutil/_common.py @@ -55,7 +55,7 @@ 'STATUS_DEAD', 'STATUS_DISK_SLEEP', 'STATUS_IDLE', 'STATUS_LOCKED', 'STATUS_RUNNING', 'STATUS_SLEEPING', 'STATUS_STOPPED', 'STATUS_SUSPENDED', 'STATUS_TRACING_STOP', 'STATUS_WAITING', 'STATUS_WAKE_KILL', - 'STATUS_WAKING', 'STATUS_ZOMBIE', + 'STATUS_WAKING', 'STATUS_ZOMBIE', 'STATUS_PARKED', # named tuples 'pconn', 'pcputimes', 'pctxsw', 'pgids', 'pio', 'pionice', 'popenfile', 'pthread', 'puids', 'sconn', 'scpustats', 'sdiskio', 'sdiskpart', @@ -100,10 +100,11 @@ STATUS_DEAD = "dead" STATUS_WAKE_KILL = "wake-kill" STATUS_WAKING = "waking" -STATUS_IDLE = "idle" # FreeBSD, macOS +STATUS_IDLE = "idle" # Linux, macOS, FreeBSD STATUS_LOCKED = "locked" # FreeBSD STATUS_WAITING = "waiting" # FreeBSD STATUS_SUSPENDED = "suspended" # NetBSD +STATUS_PARKED = "parked" # Linux # Process.connections() and psutil.net_connections() CONN_ESTABLISHED = "ESTABLISHED" diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index 85ffe66a5..ffa3a8a32 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -111,7 +111,10 @@ class IOPriority(enum.IntEnum): globals().update(IOPriority.__members__) -# taken from /fs/proc/array.c +# See: +# https://github.com/torvalds/linux/blame/master/fs/proc/array.c +# ...and (TASK_* constants): +# https://github.com/torvalds/linux/blob/master/include/linux/sched.h PROC_STATUSES = { "R": _common.STATUS_RUNNING, "S": _common.STATUS_SLEEPING, @@ -122,7 +125,9 @@ class IOPriority(enum.IntEnum): "X": _common.STATUS_DEAD, "x": _common.STATUS_DEAD, "K": _common.STATUS_WAKE_KILL, - "W": _common.STATUS_WAKING + "W": _common.STATUS_WAKING, + "I": _common.STATUS_IDLE, + "P": _common.STATUS_PARKED, } # https://github.com/torvalds/linux/blob/master/include/net/tcp_states.h From be1a35ff47fd513f6e808ac032ee7305ded1c27e Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 27 Jul 2018 17:24:10 +0200 Subject: [PATCH 098/182] disambiguate TESTFN for parallel testing --- psutil/tests/__init__.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index d293498e8..a483ecaad 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -137,7 +137,16 @@ # --- files TESTFILE_PREFIX = '$testfn' +if os.name == 'java': + # Jython disallows @ in module names + TESTFILE_PREFIX = '$psutil-test-' +else: + TESTFILE_PREFIX = '@psutil-test-' TESTFN = os.path.join(os.path.realpath(os.getcwd()), TESTFILE_PREFIX) +# Disambiguate TESTFN for parallel testing, while letting it remain a valid +# module name. +TESTFN = TESTFN + str(os.getpid()) + _TESTFN = TESTFN + '-internal' TESTFN_UNICODE = TESTFN + u("-ƒőő") ASCII_FS = sys.getfilesystemencoding().lower() in ('ascii', 'us-ascii') @@ -206,8 +215,13 @@ def attempt(exe): _testfiles_created = set() +def logstderr(s): + print(s, file=sys.stderr) + + @atexit.register -def _cleanup_files(): +def cleanup_test_files(): + logstderr("executing cleanup_test_files() atexit function") DEVNULL.close() for name in os.listdir(u('.')): if isinstance(name, unicode): @@ -215,11 +229,13 @@ def _cleanup_files(): else: prefix = TESTFILE_PREFIX if name.startswith(prefix): + logstderr("removing temporary test file %r" % name) try: safe_rmpath(name) except Exception: traceback.print_exc() for path in _testfiles_created: + logstderr("removing temporary test file %r" % path) try: safe_rmpath(path) except Exception: @@ -228,7 +244,8 @@ def _cleanup_files(): # this is executed first @atexit.register -def _cleanup_procs(): +def cleanup_test_procs(): + logstderr("executing cleanup_test_procs() atexit function") reap_children(recursive=True) From 196dcb27330c0b93f2b850d20c450216721dc8b7 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 28 Jul 2018 22:21:22 +0200 Subject: [PATCH 099/182] fix wrong reference link in doc --- docs/index.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 41b7175f9..a64448e3d 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1070,9 +1070,9 @@ Process class +------------------------------+-------------------------------+------------------------------+------------------------------+--------------------------+--------------------------+ | Linux | Windows | macOS | BSD | SunOS | AIX | +==============================+===============================+==============================+==============================+==========================+==========================+ - | :meth:`cpu_num` | :meth:`cpu_percent` | :meth:`cpu_percent` | :meth:`cpu_num` | :meth:`name` | :meth:`name` | + | :meth:`cpu_num` | :meth:`~Process.cpu_percent` | :meth:`~Process.cpu_percent` | :meth:`cpu_num` | :meth:`name` | :meth:`name` | +------------------------------+-------------------------------+------------------------------+------------------------------+--------------------------+--------------------------+ - | :meth:`cpu_percent` | :meth:`cpu_times` | :meth:`cpu_times` | :meth:`cpu_percent` | :meth:`cmdline` | :meth:`cmdline` | + | :meth:`~Process.cpu_percent` | :meth:`cpu_times` | :meth:`cpu_times` | :meth:`~Process.cpu_percent` | :meth:`cmdline` | :meth:`cmdline` | +------------------------------+-------------------------------+------------------------------+------------------------------+--------------------------+--------------------------+ | :meth:`cpu_times` | :meth:`io_counters()` | :meth:`memory_info` | :meth:`cpu_times` | :meth:`create_time` | :meth:`create_time` | +------------------------------+-------------------------------+------------------------------+------------------------------+--------------------------+--------------------------+ From 45c0ebc90e98cfe3a44d27de1a815f1fd8c0fc9c Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 29 Jul 2018 00:20:09 +0200 Subject: [PATCH 100/182] disk_io_counters() - linux: mimic iostat behavior (#1313) Fix #1244, #1305, #1312. disk_io_counters(perdisk=False) on Linux was counting disk device + partitions(s) twice --- HISTORY.rst | 2 + docs/index.rst | 4 ++ psutil/__init__.py | 3 +- psutil/_pslinux.py | 78 +++++++++++++++++++-------------- psutil/tests/test_linux.py | 89 ++++++++++++++++++++++++++------------ 5 files changed, 115 insertions(+), 61 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 602ce3d76..3dcafa3c2 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -24,6 +24,8 @@ XXXX-XX-XX MemoryError. (patch by sylvainduchesne) - 1309_: [Linux] Process.status() is unable to recognie "idle" and "parked" statuses (returns '?'). +- 1313_: [Linux] disk_io_counters() can report inflated IO counters due to + erroneously counting base disk device and its partition(s) twice. 5.4.6 ===== diff --git a/docs/index.rst b/docs/index.rst index a64448e3d..097d828dc 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -421,6 +421,8 @@ Disks cache. On Windows it may be ncessary to issue ``diskperf -y`` command from cmd.exe first in order to enable IO counters. + On diskless machines this function will return ``None`` or ``{}`` if + *perdisk* is ``True``. >>> import psutil >>> psutil.disk_io_counters() @@ -474,6 +476,8 @@ Network numbers will always be increasing or remain the same, but never decrease. ``net_io_counters.cache_clear()`` can be used to invalidate the *nowrap* cache. + On machines with no network iterfaces this function will return ``None`` or + ``{}`` if *pernic* is ``True``. >>> import psutil >>> psutil.net_io_counters() diff --git a/psutil/__init__.py b/psutil/__init__.py index 299fc1ea8..291d0d16a 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -2013,7 +2013,8 @@ def disk_io_counters(perdisk=False, nowrap=True): On recent Windows versions 'diskperf -y' command may need to be executed first otherwise this function won't find any disk. """ - rawdict = _psplatform.disk_io_counters() + kwargs = dict(perdisk=perdisk) if LINUX else {} + rawdict = _psplatform.disk_io_counters(**kwargs) if not rawdict: return {} if perdisk else None if nowrap: diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index ffa3a8a32..0aa507bd2 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -252,12 +252,12 @@ def file_flags_to_mode(flags): return mode -def get_sector_size(partition): - """Return the sector size of a partition. - Used by disk_io_counters(). - """ +def get_sector_size(name): + """Return the sector size of a storage device.""" + # Some devices may have a slash in their name (e.g. cciss/c0d0...). + name = name.replace('/', '!') try: - with open("/sys/block/%s/queue/hw_sector_size" % partition, "rt") as f: + with open("/sys/block/%s/queue/hw_sector_size" % name, "rt") as f: return int(f.read()) except (IOError, ValueError): # man iostat states that sectors are equivalent with blocks and @@ -265,6 +265,25 @@ def get_sector_size(partition): return SECTOR_SIZE_FALLBACK +def is_storage_device(name): + """Return True if the given name refers to a physical (e.g. "sda", + "nvme0n1") or virtual (e.g. "loop1", "ram") storage device. + In case name refers to a device's partition (e.g. "sda1", "nvme0n1p1") + this is supposed to return False. + """ + # Readapted from iostat source code, see: + # https://github.com/sysstat/sysstat/blob/ + # 97912938cd476645b267280069e83b1c8dc0e1c7/common.c#L208 + # Some devices may have a slash in their name (e.g. cciss/c0d0...). + name = name.replace('/', '!') + including_virtual = True + if including_virtual: + path = "/sys/block/%s" % name + else: + path = "/sys/block/%s/device" % name + return os.access(path, os.F_OK) + + @memoize def set_scputimes_ntuple(procfs_path): """Set a namedtuple of variable fields depending on the CPU times @@ -1028,32 +1047,11 @@ def net_if_stats(): disk_usage = _psposix.disk_usage -def disk_io_counters(): +def disk_io_counters(perdisk=False): """Return disk I/O statistics for every disk installed on the system as a dict of raw tuples. """ - # determine partitions we want to look for - def get_partitions(): - partitions = [] - with open_text("%s/partitions" % get_procfs_path()) as f: - lines = f.readlines()[2:] - for line in reversed(lines): - _, _, _, name = line.split() - if name[-1].isdigit(): - # we're dealing with a partition (e.g. 'sda1'); 'sda' will - # also be around but we want to omit it - partitions.append(name) - else: - if not partitions or not partitions[-1].startswith(name): - # we're dealing with a disk entity for which no - # partitions have been defined (e.g. 'sda' but - # 'sda1' was not around), see: - # https://github.com/giampaolo/psutil/issues/338 - partitions.append(name) - return partitions - retdict = {} - partitions = get_partitions() with open_text("%s/diskstats" % get_procfs_path()) as f: lines = f.readlines() for line in lines: @@ -1091,12 +1089,26 @@ def get_partitions(): else: raise ValueError("not sure how to interpret line %r" % line) - if name in partitions: - ssize = get_sector_size(name) - rbytes *= ssize - wbytes *= ssize - retdict[name] = (reads, writes, rbytes, wbytes, rtime, wtime, - reads_merged, writes_merged, busy_time) + if not perdisk and not is_storage_device(name): + # perdisk=False means we want to calculate totals so we skip + # partitions (e.g. 'sda1', 'nvme0n1p1') and only include + # base disk devices (e.g. 'sda', 'nvme0n1'). Base disks + # include a total of all their partitions + some extra size + # of their own: + # $ cat /proc/diskstats + # 259 0 sda 10485760 ... + # 259 1 sda1 5186039 ... + # 259 1 sda2 5082039 ... + # See: + # https://github.com/giampaolo/psutil/pull/1313 + continue + + ssize = get_sector_size(name) + rbytes *= ssize + wbytes *= ssize + retdict[name] = (reads, writes, rbytes, wbytes, rtime, wtime, + reads_merged, writes_merged, busy_time) + return retdict diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index 9ea59b617..761f79047 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -986,15 +986,10 @@ def test_disk_io_counters_kernel_2_4_mocked(self): # Tests /proc/diskstats parsing format for 2.4 kernels, see: # https://github.com/giampaolo/psutil/issues/767 with mock_open_content( - '/proc/partitions', - textwrap.dedent("""\ - major minor #blocks name - - 8 0 488386584 hda - """)): - with mock_open_content( - '/proc/diskstats', - " 3 0 1 hda 2 3 4 5 6 7 8 9 10 11 12"): + '/proc/diskstats', + " 3 0 1 hda 2 3 4 5 6 7 8 9 10 11 12"): + with mock.patch('psutil._pslinux.is_storage_device', + return_value=True): ret = psutil.disk_io_counters(nowrap=False) self.assertEqual(ret.read_count, 1) self.assertEqual(ret.read_merged_count, 2) @@ -1011,15 +1006,10 @@ def test_disk_io_counters_kernel_2_6_full_mocked(self): # lines reporting all metrics: # https://github.com/giampaolo/psutil/issues/767 with mock_open_content( - '/proc/partitions', - textwrap.dedent("""\ - major minor #blocks name - - 8 0 488386584 hda - """)): - with mock_open_content( - '/proc/diskstats', - " 3 0 hda 1 2 3 4 5 6 7 8 9 10 11"): + '/proc/diskstats', + " 3 0 hda 1 2 3 4 5 6 7 8 9 10 11"): + with mock.patch('psutil._pslinux.is_storage_device', + return_value=True): ret = psutil.disk_io_counters(nowrap=False) self.assertEqual(ret.read_count, 1) self.assertEqual(ret.read_merged_count, 2) @@ -1038,15 +1028,10 @@ def test_disk_io_counters_kernel_2_6_limited_mocked(self): # (instead of a disk). See: # https://github.com/giampaolo/psutil/issues/767 with mock_open_content( - '/proc/partitions', - textwrap.dedent("""\ - major minor #blocks name - - 8 0 488386584 hda - """)): - with mock_open_content( - '/proc/diskstats', - " 3 1 hda 1 2 3 4"): + '/proc/diskstats', + " 3 1 hda 1 2 3 4"): + with mock.patch('psutil._pslinux.is_storage_device', + return_value=True): ret = psutil.disk_io_counters(nowrap=False) self.assertEqual(ret.read_count, 1) self.assertEqual(ret.read_bytes, 2 * SECTOR_SIZE) @@ -1059,6 +1044,56 @@ def test_disk_io_counters_kernel_2_6_limited_mocked(self): self.assertEqual(ret.write_time, 0) self.assertEqual(ret.busy_time, 0) + def test_disk_io_counters_include_partitions(self): + # Make sure that when perdisk=True disk partitions are returned, + # see: + # https://github.com/giampaolo/psutil/pull/1313#issuecomment-408626842 + with mock_open_content( + '/proc/diskstats', + textwrap.dedent("""\ + 3 0 nvme0n1 1 2 3 4 5 6 7 8 9 10 11 + 3 0 nvme0n1p1 1 2 3 4 5 6 7 8 9 10 11 + """)): + with mock.patch('psutil._pslinux.is_storage_device', + return_value=False): + ret = psutil.disk_io_counters(perdisk=True, nowrap=False) + self.assertEqual(len(ret), 2) + self.assertEqual(ret['nvme0n1'].read_count, 1) + self.assertEqual(ret['nvme0n1p1'].read_count, 1) + self.assertEqual(ret['nvme0n1'].write_count, 5) + self.assertEqual(ret['nvme0n1p1'].write_count, 5) + + def test_disk_io_counters_exclude_partitions(self): + # Make sure that when perdisk=False partitions (e.g. 'sda1', + # 'nvme0n1p1') are skipped and not included in the total count. + # https://github.com/giampaolo/psutil/pull/1313#issuecomment-408626842 + with mock_open_content( + '/proc/diskstats', + textwrap.dedent("""\ + 3 0 nvme0n1 1 2 3 4 5 6 7 8 9 10 11 + 3 0 nvme0n1p1 1 2 3 4 5 6 7 8 9 10 11 + """)): + with mock.patch('psutil._pslinux.is_storage_device', + return_value=False): + ret = psutil.disk_io_counters(perdisk=False, nowrap=False) + self.assertIsNone(ret) + + # + def is_storage_device(name): + return name == 'nvme0n1' + + with mock_open_content( + '/proc/diskstats', + textwrap.dedent("""\ + 3 0 nvme0n1 1 2 3 4 5 6 7 8 9 10 11 + 3 0 nvme0n1p1 1 2 3 4 5 6 7 8 9 10 11 + """)): + with mock.patch('psutil._pslinux.is_storage_device', + create=True, side_effect=is_storage_device): + ret = psutil.disk_io_counters(perdisk=False, nowrap=False) + self.assertEqual(ret.read_count, 1) + self.assertEqual(ret.write_count, 5) + # ===================================================================== # --- misc From 6b4afea8ed6bb94611ec806f7695bd8d7d5f41e9 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 29 Jul 2018 01:30:42 +0200 Subject: [PATCH 101/182] #1313 remove test which no longer makes sense --- psutil/tests/test_system.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/psutil/tests/test_system.py b/psutil/tests/test_system.py index 18cbb5d8b..f9006ce81 100755 --- a/psutil/tests/test_system.py +++ b/psutil/tests/test_system.py @@ -712,12 +712,6 @@ def check_ntuple(nt): for key in ret: assert key, key check_ntuple(ret[key]) - if LINUX and key[-1].isdigit(): - # if 'sda1' is listed 'sda' shouldn't, see: - # https://github.com/giampaolo/psutil/issues/338 - while key[-1].isdigit(): - key = key[:-1] - self.assertNotIn(key, ret.keys()) def test_disk_io_counters_no_disks(self): # Emulate a case where no disks are installed, see: From 96363492d1455ad469277675583383690fbba080 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 1 Aug 2018 20:53:11 +0200 Subject: [PATCH 102/182] fix #1305 / disk_io_counters() / Linux: assume SECTOR_SIZE is a fixed 512 --- HISTORY.rst | 1 + psutil/_pslinux.py | 33 ++++++++++++++++----------------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 3dcafa3c2..8026def74 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -22,6 +22,7 @@ XXXX-XX-XX - 1279_: [Linux, macOS, BSD] net_if_stats() may return ENODEV. - 1294_: [Windows] psutil.Process().connections() may sometime fail with MemoryError. (patch by sylvainduchesne) +- 1305_: [Linux] disk_io_stats() may report inflated r/w bytes values. - 1309_: [Linux] Process.status() is unable to recognie "idle" and "parked" statuses (returns '?'). - 1313_: [Linux] disk_io_counters() can report inflated IO counters due to diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index 0aa507bd2..54fa0d893 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -88,7 +88,20 @@ # speedup, see: https://github.com/giampaolo/psutil/issues/708 BIGFILE_BUFFERING = -1 if PY3 else 8192 LITTLE_ENDIAN = sys.byteorder == 'little' -SECTOR_SIZE_FALLBACK = 512 + +# "man iostat" states that sectors are equivalent with blocks and have +# a size of 512 bytes. Despite this value can be queried at runtime +# via /sys/block/{DISK}/queue/hw_sector_size and results may vary +# between 1k, 2k, or 4k... 512 appears to be a magic constant used +# throughout Linux source code: +# * https://stackoverflow.com/a/38136179/376587 +# * https://lists.gt.net/linux/kernel/2241060 +# * https://github.com/giampaolo/psutil/issues/1305 +# * https://github.com/torvalds/linux/blob/ +# 4f671fe2f9523a1ea206f63fe60a7c7b3a56d5c7/include/linux/bio.h#L99 +# * https://lkml.org/lkml/2015/8/17/234 +DISK_SECTOR_SIZE = 512 + if enum is None: AF_LINK = socket.AF_PACKET else: @@ -252,19 +265,6 @@ def file_flags_to_mode(flags): return mode -def get_sector_size(name): - """Return the sector size of a storage device.""" - # Some devices may have a slash in their name (e.g. cciss/c0d0...). - name = name.replace('/', '!') - try: - with open("/sys/block/%s/queue/hw_sector_size" % name, "rt") as f: - return int(f.read()) - except (IOError, ValueError): - # man iostat states that sectors are equivalent with blocks and - # have a size of 512 bytes since 2.4 kernels. - return SECTOR_SIZE_FALLBACK - - def is_storage_device(name): """Return True if the given name refers to a physical (e.g. "sda", "nvme0n1") or virtual (e.g. "loop1", "ram") storage device. @@ -1103,9 +1103,8 @@ def disk_io_counters(perdisk=False): # https://github.com/giampaolo/psutil/pull/1313 continue - ssize = get_sector_size(name) - rbytes *= ssize - wbytes *= ssize + rbytes *= DISK_SECTOR_SIZE + wbytes *= DISK_SECTOR_SIZE retdict[name] = (reads, writes, rbytes, wbytes, rtime, wtime, reads_merged, writes_merged, busy_time) From a1f37298204d7dee2881fa0dab4665c243689982 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 1 Aug 2018 21:16:55 +0200 Subject: [PATCH 103/182] update is_storage_device() docstring --- psutil/_pslinux.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index 54fa0d893..7e01bb731 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -266,10 +266,10 @@ def file_flags_to_mode(flags): def is_storage_device(name): - """Return True if the given name refers to a physical (e.g. "sda", - "nvme0n1") or virtual (e.g. "loop1", "ram") storage device. - In case name refers to a device's partition (e.g. "sda1", "nvme0n1p1") - this is supposed to return False. + """Return True if the given name refers to a root device (e.g. + "sda", "nvme0n1") as opposed to a logical partition (e.g. "sda1", + "nvme0n1p1"). If name is a virtual device (e.g. "loop1", "ram") + return True. """ # Readapted from iostat source code, see: # https://github.com/sysstat/sysstat/blob/ From 892e70af56742708cbb38e311c6b4b916f0a0a46 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 1 Aug 2018 21:31:27 +0200 Subject: [PATCH 104/182] remove old test --- psutil/tests/test_linux.py | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index 761f79047..18d9e07a0 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -1257,25 +1257,6 @@ def test_procfs_path(self): psutil.PROCFS_PATH = "/proc" os.rmdir(tdir) - def test_sector_size_mock(self): - # Test SECTOR_SIZE fallback in case 'hw_sector_size' file - # does not exist. - def open_mock(name, *args, **kwargs): - if PY3 and isinstance(name, bytes): - name = name.decode() - if "hw_sector_size" in name: - flag.append(None) - raise IOError(errno.ENOENT, '') - else: - return orig_open(name, *args, **kwargs) - - flag = [] - orig_open = open - patch_point = 'builtins.open' if PY3 else '__builtin__.open' - with mock.patch(patch_point, side_effect=open_mock): - psutil.disk_io_counters() - assert flag - def test_issue_687(self): # In case of thread ID: # - pid_exists() is supposed to return False From 0352a00e863afd168dfcf5f78f0b7c8eb1a70926 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Schoentgen?= Date: Wed, 8 Aug 2018 19:36:31 +0200 Subject: [PATCH 105/182] Fix DeprecationWarning: invalid escape sequence (#1318) --- psutil/_psaix.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/psutil/_psaix.py b/psutil/_psaix.py index 662f306c3..b402a188f 100644 --- a/psutil/_psaix.py +++ b/psutil/_psaix.py @@ -269,7 +269,7 @@ def net_if_stats(): stdout, stderr = [x.decode(sys.stdout.encoding) for x in (stdout, stderr)] if p.returncode == 0: - re_result = re.search("Running: (\d+) Mbps.*?(\w+) Duplex", stdout) + re_result = re.search(r"Running: (\d+) Mbps.*?(\w+) Duplex", stdout) if re_result is not None: speed = int(re_result.group(1)) duplex = re_result.group(2) @@ -534,7 +534,7 @@ def open_files(self): for x in (stdout, stderr)] if "no such process" in stderr.lower(): raise NoSuchProcess(self.pid, self._name) - procfiles = re.findall("(\d+): S_IFREG.*\s*.*name:(.*)\n", stdout) + procfiles = re.findall(r"(\d+): S_IFREG.*\s*.*name:(.*)\n", stdout) retlist = [] for fd, path in procfiles: path = path.strip() From b739e13bcf59b88d9b6b4e5e71391d600eed8e4c Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 12 Aug 2018 10:44:33 +0200 Subject: [PATCH 106/182] fix typo --- HISTORY.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index 8026def74..440392e19 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -23,7 +23,7 @@ XXXX-XX-XX - 1294_: [Windows] psutil.Process().connections() may sometime fail with MemoryError. (patch by sylvainduchesne) - 1305_: [Linux] disk_io_stats() may report inflated r/w bytes values. -- 1309_: [Linux] Process.status() is unable to recognie "idle" and "parked" +- 1309_: [Linux] Process.status() is unable to recognize "idle" and "parked" statuses (returns '?'). - 1313_: [Linux] disk_io_counters() can report inflated IO counters due to erroneously counting base disk device and its partition(s) twice. From b9e60739aebaf824c692efedeb9cf9437c26ac49 Mon Sep 17 00:00:00 2001 From: Lawrence Ye Date: Mon, 13 Aug 2018 16:27:30 +0800 Subject: [PATCH 107/182] make disk_io_counters more robust (#1324) --- psutil/_pslinux.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index 7e01bb731..1669c4538 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -1052,8 +1052,23 @@ def disk_io_counters(perdisk=False): system as a dict of raw tuples. """ retdict = {} - with open_text("%s/diskstats" % get_procfs_path()) as f: - lines = f.readlines() + if os.path.exists("%s/diskstats" % get_procfs_path()): + with open_text("%s/diskstats" % get_procfs_path()) as f: + lines = f.readlines() + else: + # Try to use /sys/block/*/stat for disk_io_counters + # if /process/diskstats doesn't exist + lines = [] + for block in os.listdir('/sys/block'): + for root, _dirs, files in os.walk( + os.path.join('/sys/block', block)): + if 'stat' in files: + with open_text(os.path.join(root, 'stat')) as f: + line = f.readline() + # Let's just set major and minor device number + # to zero since we don't care about that + lines.append( + "0 0 %s %s" % (os.path.basename(root), line)) for line in lines: # OK, this is a bit confusing. The format of /proc/diskstats can # have 3 variations. From 0806efa9af59e1a0de5828af759b9afc47dfda2e Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 13 Aug 2018 11:40:05 +0200 Subject: [PATCH 108/182] #1321: refactoring --- CREDITS | 4 ++ HISTORY.rst | 3 ++ psutil/_pslinux.py | 91 ++++++++++++++++++++++---------------- psutil/tests/test_linux.py | 2 +- 4 files changed, 60 insertions(+), 40 deletions(-) diff --git a/CREDITS b/CREDITS index edf1799dd..27a4ed148 100644 --- a/CREDITS +++ b/CREDITS @@ -551,3 +551,7 @@ I: 1284 N: sylvainduchesne W: https://github.com/sylvainduchesne I: 1294 + +N: Lawrence Ye +W: https://github.com/LEAFERx +I: 1321 diff --git a/HISTORY.rst b/HISTORY.rst index 440392e19..e9162de5b 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -12,6 +12,9 @@ XXXX-XX-XX - 1286_: [macOS] psutil.OSX constant is now deprecated in favor of new psutil.MACOS. - 1309_: [Linux] added psutil.STATUS_PARKED constant for Process.status(). +- 1321_: [Linux] add disk_io_counters() dual implementation relying on + /sys/block filesystem in case /proc/diskstats is not available. (patch by + Lawrence Ye) **Bug fixes** diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index 1669c4538..63b4ded1e 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -1051,25 +1051,7 @@ def disk_io_counters(perdisk=False): """Return disk I/O statistics for every disk installed on the system as a dict of raw tuples. """ - retdict = {} - if os.path.exists("%s/diskstats" % get_procfs_path()): - with open_text("%s/diskstats" % get_procfs_path()) as f: - lines = f.readlines() - else: - # Try to use /sys/block/*/stat for disk_io_counters - # if /process/diskstats doesn't exist - lines = [] - for block in os.listdir('/sys/block'): - for root, _dirs, files in os.walk( - os.path.join('/sys/block', block)): - if 'stat' in files: - with open_text(os.path.join(root, 'stat')) as f: - line = f.readline() - # Let's just set major and minor device number - # to zero since we don't care about that - lines.append( - "0 0 %s %s" % (os.path.basename(root), line)) - for line in lines: + def read_procfs(): # OK, this is a bit confusing. The format of /proc/diskstats can # have 3 variations. # On Linux 2.4 each line has always 15 fields, e.g.: @@ -1083,27 +1065,58 @@ def disk_io_counters(perdisk=False): # See: # https://www.kernel.org/doc/Documentation/iostats.txt # https://www.kernel.org/doc/Documentation/ABI/testing/procfs-diskstats - fields = line.split() - fields_len = len(fields) - if fields_len == 15: - # Linux 2.4 - name = fields[3] - reads = int(fields[2]) - (reads_merged, rbytes, rtime, writes, writes_merged, - wbytes, wtime, _, busy_time, _) = map(int, fields[4:14]) - elif fields_len == 14: - # Linux 2.6+, line referring to a disk - name = fields[2] - (reads, reads_merged, rbytes, rtime, writes, writes_merged, - wbytes, wtime, _, busy_time, _) = map(int, fields[3:14]) - elif fields_len == 7: - # Linux 2.6+, line referring to a partition - name = fields[2] - reads, rbytes, writes, wbytes = map(int, fields[3:]) - rtime = wtime = reads_merged = writes_merged = busy_time = 0 - else: - raise ValueError("not sure how to interpret line %r" % line) + with open_text("%s/diskstats" % get_procfs_path()) as f: + lines = f.readlines() + for line in lines: + fields = line.split() + flen = len(fields) + if flen == 15: + # Linux 2.4 + name = fields[3] + reads = int(fields[2]) + (reads_merged, rbytes, rtime, writes, writes_merged, + wbytes, wtime, _, busy_time, _) = map(int, fields[4:14]) + elif flen == 14: + # Linux 2.6+, line referring to a disk + name = fields[2] + (reads, reads_merged, rbytes, rtime, writes, writes_merged, + wbytes, wtime, _, busy_time, _) = map(int, fields[3:14]) + elif flen == 7: + # Linux 2.6+, line referring to a partition + name = fields[2] + reads, rbytes, writes, wbytes = map(int, fields[3:]) + rtime = wtime = reads_merged = writes_merged = busy_time = 0 + else: + raise ValueError("not sure how to interpret line %r" % line) + yield (name, reads, writes, rbytes, wbytes, rtime, wtime, + reads_merged, writes_merged, busy_time) + def read_sysfs(): + for block in os.listdir('/sys/block'): + for root, _, files in os.walk(os.path.join('/sys/block', block)): + if 'stat' not in files: + continue + with open_text(os.path.join(root, 'stat')) as f: + fields = f.read().strip().split() + name = os.path.basename(root) + (reads, reads_merged, rbytes, rtime, writes, writes_merged, + wbytes, wtime, _, busy_time, _) = map(int, fields) + yield (name, reads, writes, rbytes, wbytes, rtime, + wtime, reads_merged, writes_merged, busy_time) + + if os.path.exists('%s/diskstats' % get_procfs_path()): + gen = read_procfs() + elif os.path.exists('/sys/block'): + gen = read_sysfs() + else: + raise NotImplementedError( + "%s/diskstats nor /sys/block filesystem are available on this " + "system" % get_procfs_path()) + + retdict = {} + for entry in gen: + (name, reads, writes, rbytes, wbytes, rtime, wtime, reads_merged, + writes_merged, busy_time) = entry if not perdisk and not is_storage_device(name): # perdisk=False means we want to calculate totals so we skip # partitions (e.g. 'sda1', 'nvme0n1p1') and only include diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index 18d9e07a0..0216eabb7 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -1250,7 +1250,7 @@ def test_procfs_path(self): self.assertRaises(IOError, psutil.net_connections) self.assertRaises(IOError, psutil.net_io_counters) self.assertRaises(IOError, psutil.net_if_stats) - self.assertRaises(IOError, psutil.disk_io_counters) + # self.assertRaises(IOError, psutil.disk_io_counters) self.assertRaises(IOError, psutil.disk_partitions) self.assertRaises(psutil.NoSuchProcess, psutil.Process) finally: From 67e8874a8bead8637948b770c806dc5c10f80bd9 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 13 Aug 2018 13:06:45 +0200 Subject: [PATCH 109/182] #1321 add unit tests --- psutil/tests/test_linux.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index 0216eabb7..4e652a9d1 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -1094,6 +1094,26 @@ def is_storage_device(name): self.assertEqual(ret.read_count, 1) self.assertEqual(ret.write_count, 5) + def test_disk_io_counters_sysfs(self): + def exists(path): + if path == '/proc/diskstats': + return False + return True + + wprocfs = psutil.disk_io_counters(perdisk=True) + with mock.patch('psutil._pslinux.os.path.exists', + create=True, side_effect=exists): + wsysfs = psutil.disk_io_counters(perdisk=True) + self.assertEqual(len(wprocfs), len(wsysfs)) + + def test_disk_io_counters_not_impl(self): + def exists(path): + return False + + with mock.patch('psutil._pslinux.os.path.exists', + create=True, side_effect=exists): + self.assertRaises(NotImplementedError, psutil.disk_io_counters) + # ===================================================================== # --- misc From 32cefd018133443c60a280488520b3d096d819b5 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 13 Aug 2018 13:53:05 +0200 Subject: [PATCH 110/182] fix failing linux tests --- psutil/_psaix.py | 3 ++- psutil/tests/test_contracts.py | 18 ++++-------------- 2 files changed, 6 insertions(+), 15 deletions(-) diff --git a/psutil/_psaix.py b/psutil/_psaix.py index b402a188f..7ba212dbf 100644 --- a/psutil/_psaix.py +++ b/psutil/_psaix.py @@ -269,7 +269,8 @@ def net_if_stats(): stdout, stderr = [x.decode(sys.stdout.encoding) for x in (stdout, stderr)] if p.returncode == 0: - re_result = re.search(r"Running: (\d+) Mbps.*?(\w+) Duplex", stdout) + re_result = re.search( + r"Running: (\d+) Mbps.*?(\w+) Duplex", stdout) if re_result is not None: speed = int(re_result.group(1)) duplex = re_result.group(2) diff --git a/psutil/tests/test_contracts.py b/psutil/tests/test_contracts.py index ac424b54f..da1c0c856 100755 --- a/psutil/tests/test_contracts.py +++ b/psutil/tests/test_contracts.py @@ -289,16 +289,6 @@ class TestFetchAllProcesses(unittest.TestCase): some sanity checks against Process API's returned values. """ - def setUp(self): - if POSIX: - import pwd - import grp - users = pwd.getpwall() - groups = grp.getgrall() - self.all_uids = set([x.pw_uid for x in users]) - self.all_usernames = set([x.pw_name for x in users]) - self.all_gids = set([x.gr_gid for x in groups]) - def test_fetch_all(self): valid_procs = 0 excluded_names = set([ @@ -425,7 +415,6 @@ def uids(self, ret, proc): for uid in ret: self.assertIsInstance(uid, int) self.assertGreaterEqual(uid, 0) - self.assertIn(uid, self.all_uids) def gids(self, ret, proc): assert is_namedtuple(ret) @@ -435,13 +424,10 @@ def gids(self, ret, proc): self.assertIsInstance(gid, int) if not MACOS and not NETBSD: self.assertGreaterEqual(gid, 0) - self.assertIn(gid, self.all_gids) def username(self, ret, proc): self.assertIsInstance(ret, str) assert ret - if POSIX: - self.assertIn(ret, self.all_usernames) def status(self, ret, proc): self.assertIsInstance(ret, str) @@ -526,6 +512,10 @@ def memory_full_info(self, ret, proc): value = getattr(ret, name) self.assertIsInstance(value, (int, long)) self.assertGreaterEqual(value, 0, msg=(name, value)) + if LINUX and name in ('vms', 'data'): + # On Linux there are processes (e.g. 'goa-daemon') whose + # VMS is incredibly high for some reason. + continue self.assertLessEqual(value, total, msg=(name, value, total)) if LINUX: From 5bd44f8afcecbfb0db479ce230c790fc2c56569a Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 13 Aug 2018 14:12:04 +0200 Subject: [PATCH 111/182] fix #1323: [Linux] sensors_temperatures() may fail with ValueError --- HISTORY.rst | 1 + psutil/_pslinux.py | 18 +++++++++++++----- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index e9162de5b..7532112df 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -30,6 +30,7 @@ XXXX-XX-XX statuses (returns '?'). - 1313_: [Linux] disk_io_counters() can report inflated IO counters due to erroneously counting base disk device and its partition(s) twice. +- 1323_: [Linux] sensors_temperatures() may fail with ValueError. 5.4.6 ===== diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index 63b4ded1e..df624de32 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -318,7 +318,7 @@ def cat(fname, fallback=_DEFAULT, binary=True): try: with open_binary(fname) if binary else open_text(fname) as f: return f.read().strip() - except IOError: + except (IOError, OSError): if fallback is not _DEFAULT: return fallback else: @@ -1199,13 +1199,15 @@ def sensors_temperatures(): current = float(cat(path)) / 1000.0 path = os.path.join(os.path.dirname(base), 'name') unit_name = cat(path, binary=False) - except (IOError, OSError) as err: + except (IOError, OSError, ValueError) as err: # A lot of things can go wrong here, so let's just skip the - # whole entry. + # whole entry. Sure thing is Linux's /sys/class/hwmon really + # is a stinky broken mess. # https://github.com/giampaolo/psutil/issues/1009 # https://github.com/giampaolo/psutil/issues/1101 # https://github.com/giampaolo/psutil/issues/1129 # https://github.com/giampaolo/psutil/issues/1245 + # https://github.com/giampaolo/psutil/issues/1323 warnings.warn("ignoring %r for file %r" % (err, path), RuntimeWarning) continue @@ -1215,9 +1217,15 @@ def sensors_temperatures(): label = cat(base + '_label', fallback='', binary=False) if high is not None: - high = float(high) / 1000.0 + try: + high = float(high) / 1000.0 + except ValueError: + high = None if critical is not None: - critical = float(critical) / 1000.0 + try: + critical = float(critical) / 1000.0 + except ValueError: + critical = None ret[unit_name].append((label, current, high, critical)) From 4b71736b72b6b68a81dfb22d2df431c57ef1fde1 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 13 Aug 2018 22:02:29 +0200 Subject: [PATCH 112/182] setup.py: add py 3.7 --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index adc7a0e2d..9815616dc 100755 --- a/setup.py +++ b/setup.py @@ -322,6 +322,7 @@ def main(): 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: Implementation :: CPython', 'Programming Language :: Python :: Implementation :: PyPy', 'Programming Language :: Python', From 2820f92554cbce52afdab3710413e26975cddd5e Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 14 Aug 2018 01:37:17 +0200 Subject: [PATCH 113/182] OSX / SMC / sensors: revert #1284 (#1325) * setup.py: add py 3.7 * Revert OSX sensors code due to copyright constraints It turns out code contributed in PR #1284 was using parts of a source code released by Apple [1] which uses GPL license which is incompatible with psutil's license (BSD). [1] https://gist.github.com/edvakf/4049362 --- HISTORY.rst | 2 - docs/index.rst | 4 - psutil/_psosx.py | 141 ------------------- psutil/_psutil_osx.c | 62 --------- psutil/arch/osx/smc.c | 238 --------------------------------- psutil/arch/osx/smc.h | 79 ----------- psutil/tests/test_contracts.py | 4 +- setup.py | 2 +- 8 files changed, 3 insertions(+), 529 deletions(-) delete mode 100644 psutil/arch/osx/smc.c delete mode 100644 psutil/arch/osx/smc.h diff --git a/HISTORY.rst b/HISTORY.rst index 7532112df..e295436ab 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -7,8 +7,6 @@ XXXX-XX-XX **Enhancements** -- 1284_: [macOS] added support for sensors_temperatures() and sensors_fans(). - (patch by Alex Manuskin) - 1286_: [macOS] psutil.OSX constant is now deprecated in favor of new psutil.MACOS. - 1309_: [Linux] added psutil.STATUS_PARKED constant for Process.status(). diff --git a/docs/index.rst b/docs/index.rst index 097d828dc..c70d2f497 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -706,8 +706,6 @@ Sensors .. versionadded:: 5.1.0 - .. versionchanged:: 5.5.0: added macOS support - .. warning:: this API is experimental. Backward incompatible changes may occur if @@ -732,8 +730,6 @@ Sensors .. versionadded:: 5.2.0 - .. versionchanged:: 5.5.0: added macOS support - .. warning:: this API is experimental. Backward incompatible changes may occur if diff --git a/psutil/_psosx.py b/psutil/_psosx.py index d059449af..fbfedf3ea 100644 --- a/psutil/_psosx.py +++ b/psutil/_psosx.py @@ -4,7 +4,6 @@ """macOS platform implementation.""" -import collections import contextlib import errno import functools @@ -63,108 +62,6 @@ cext.SZOMB: _common.STATUS_ZOMBIE, } -SMC_TEMPERATURES = ( - # --- CPU - ("CPU", "TCXC", "PECI CPU"), - ("CPU", "TCXc", "PECI CPU"), - ("CPU", "TC0P", "CPU 1 Proximity"), - ("CPU", "TC0H", "CPU 1 Heatsink"), - ("CPU", "TC0D", "CPU 1 Package"), - ("CPU", "TC0E", "CPU 1"), - ("CPU", "TC1C", "CPU Core 1"), - ("CPU", "TC2C", "CPU Core 2"), - ("CPU", "TC3C", "CPU Core 3"), - ("CPU", "TC4C", "CPU Core 4"), - ("CPU", "TC5C", "CPU Core 5"), - ("CPU", "TC6C", "CPU Core 6"), - ("CPU", "TC7C", "CPU Core 7"), - ("CPU", "TC8C", "CPU Core 8"), - ("CPU", "TCAH", "CPU 1 Heatsink Alt."), - ("CPU", "TCAD", "CPU 1 Package Alt."), - ("CPU", "TC1P", "CPU 2 Proximity"), - ("CPU", "TC1H", "CPU 2 Heatsink"), - ("CPU", "TC1D", "CPU 2 Package"), - ("CPU", "TC1E", "CPU 2"), - ("CPU", "TCBH", "CPU 2 Heatsink Alt."), - ("CPU", "TCBD", "CPU 2 Package Alt."), - - ("CPU", "TCSC", "PECI SA"), - ("CPU", "TCSc", "PECI SA"), - ("CPU", "TCSA", "PECI SA"), - - # --- GPU - ("GPU", "TCGC", "PECI GPU"), - ("GPU", "TCGc", "PECI GPU"), - ("GPU", "TG0P", "GPU Proximity"), - ("GPU", "TG0D", "GPU Die"), - ("GPU", "TG1D", "GPU Die"), - ("GPU", "TG0H", "GPU Heatsink"), - ("GPU", "TG1H", "GPU Heatsink"), - - # --- Memory - ("Memory", "Ts0S", "Memory Proximity"), - ("Memory", "TM0P", "Mem Bank A1"), - ("Memory", "TM1P", "Mem Bank A2"), - ("Memory", "TM8P", "Mem Bank B1"), - ("Memory", "TM9P", "Mem Bank B2"), - ("Memory", "TM0S", "Mem Module A1"), - ("Memory", "TM1S", "Mem Module A2"), - ("Memory", "TM8S", "Mem Module B1"), - ("Memory", "TM9S", "Mem Module B2"), - - # --- HDD - ("HDD", "TH0P", "HDD Bay 1"), - ("HDD", "TH1P", "HDD Bay 2"), - ("HDD", "TH2P", "HDD Bay 3"), - ("HDD", "TH3P", "HDD Bay 4"), - - # --- Battery - ("Battery", "TB0T", "Battery TS_MAX"), - ("Battery", "TB1T", "Battery 1"), - ("Battery", "TB2T", "Battery 2"), - ("Battery", "TB3T", "Battery"), - - # --- Others - ("Others", "TN0D", "Northbridge Die"), - ("Others", "TN0P", "Northbridge Proximity 1"), - ("Others", "TN1P", "Northbridge Proximity 2"), - ("Others", "TN0C", "MCH Die"), - ("Others", "TN0H", "MCH Heatsink"), - ("Others", "TP0D", "PCH Die"), - ("Others", "TPCD", "PCH Die"), - ("Others", "TP0P", "PCH Proximity"), - - ("Others", "TA0P", "Airflow 1"), - ("Others", "TA1P", "Airflow 2"), - ("Others", "Th0H", "Heatpipe 1"), - ("Others", "Th1H", "Heatpipe 2"), - ("Others", "Th2H", "Heatpipe 3"), - - ("Others", "Tm0P", "Mainboard Proximity"), - ("Others", "Tp0P", "Powerboard Proximity"), - ("Others", "Ts0P", "Palm Rest"), - ("Others", "Tb0P", "BLC Proximity"), - - ("Others", "TL0P", "LCD Proximity"), - ("Others", "TW0P", "Airport Proximity"), - ("Others", "TO0P", "Optical Drive"), - - ("Others", "Tp0P", "Power Supply 1"), - ("Others", "Tp0C", "Power Supply 1 Alt."), - ("Others", "Tp1P", "Power Supply 2"), - ("Others", "Tp1C", "Power Supply 2 Alt."), - ("Others", "Tp2P", "Power Supply 3"), - ("Others", "Tp3P", "Power Supply 4"), - ("Others", "Tp4P", "Power Supply 5"), - ("Others", "Tp5P", "Power Supply 6"), - - ("Others", "TS0C", "Expansion Slots"), - ("Others", "TA0S", "PCI Slot 1 Pos 1"), - ("Others", "TA1S", "PCI Slot 1 Pos 2"), - ("Others", "TA2S", "PCI Slot 2 Pos 1"), - ("Others", "TA3S", "PCI Slot 2 Pos 2"), -) - kinfo_proc_map = dict( ppid=0, ruid=1, @@ -315,34 +212,6 @@ def disk_partitions(all=False): # ===================================================================== -def sensors_temperatures(): - """Returns a dictionary of regions of temperature sensors: - CPU/GPU/Memory/HDD/Battery/Others. - Each entry contains a list of results of all the SMC keys successfully - polled from the system. - References for SMC keys and meaning: - - * https://stackoverflow.com/questions/28568775/ - description-for-apples-smc-keys/31033665#31033665 - - * https://github.com/Chris911/iStats/blob/ - 09b159f85a9481b59f347a37259f6d272f65cc05/lib/iStats/smc.rb - - * http://web.archive.org/web/20140714090133/http://www.parhelia.ch:80/ - blog/statics/k3_keys.html - """ - # TODO: this should be rewritten in C: - # https://github.com/giampaolo/psutil/pull/1284#issuecomment-399480983 - ret = collections.defaultdict(list) - for group, key, label in SMC_TEMPERATURES: - result = cext.smc_get_temperature(key) - result = round(result, 1) - if result <= 0: - continue - ret[group].append((label, result, None, None)) - return dict(ret) - - def sensors_battery(): """Return battery information.""" try: @@ -360,16 +229,6 @@ def sensors_battery(): return _common.sbattery(percent, secsleft, power_plugged) -def sensors_fans(): - """Return fans speed information. - """ - ret = collections.defaultdict(list) - rawlist = cext.sensors_fans() - for fan in rawlist: - ret["Fans"].append(_common.sfan(fan[0], fan[1])) - return dict(ret) - - # ===================================================================== # --- network # ===================================================================== diff --git a/psutil/_psutil_osx.c b/psutil/_psutil_osx.c index e3250caba..18dbe8931 100644 --- a/psutil/_psutil_osx.c +++ b/psutil/_psutil_osx.c @@ -44,7 +44,6 @@ #include "_psutil_common.h" #include "_psutil_posix.h" #include "arch/osx/process_info.h" -#include "arch/osx/smc.h" #define PSUTIL_TV2DOUBLE(t) ((t).tv_sec + (t).tv_usec / 1000000.0) @@ -897,62 +896,6 @@ psutil_boot_time(PyObject *self, PyObject *args) { return Py_BuildValue("f", (float)boot_time); } -/* - * Return a Python float indicating the value of the temperature - * measured by an SMC key - */ -static PyObject * -psutil_smc_get_temperature(PyObject *self, PyObject *args) { - char* key; - float temp; - - if (! PyArg_ParseTuple(args, "s", &key)) { - return NULL; - } - temp = SMCGetTemperature(key); - return Py_BuildValue("d", temp); -} - - -/* - * Return a Python list of tuples of fan label and speed - */ -static PyObject * -psutil_sensors_fans(PyObject *self, PyObject *args) { - int key; - int speed; - char fan[7]; - int fan_count; - PyObject *py_tuple = NULL; - PyObject *py_retlist = PyList_New(0); - - if (py_retlist == NULL) - return NULL; - - fan_count = SMCGetFanNumber(SMC_KEY_FAN_NUM); - if (fan_count < 0) - fan_count = 0; - - for (key = 0; key < fan_count; key++) { - sprintf(fan, "Fan %d", key); - speed = SMCGetFanSpeed(key); - if (speed < 0) - continue; - py_tuple = Py_BuildValue("(si)", fan, speed); - if (!py_tuple) - goto error; - if (PyList_Append(py_retlist, py_tuple)) - goto error; - Py_XDECREF(py_tuple); - } - - return py_retlist; - -error: - Py_XDECREF(py_tuple); - Py_DECREF(py_retlist); - return NULL; -} /* * Return a list of tuples including device, mount point and fs type @@ -1885,7 +1828,6 @@ psutil_cpu_stats(PyObject *self, PyObject *args) { } - /* * Return battery information. */ @@ -2038,10 +1980,6 @@ PsutilMethods[] = { "Return currently connected users as a list of tuples"}, {"cpu_stats", psutil_cpu_stats, METH_VARARGS, "Return CPU statistics"}, - {"smc_get_temperature", psutil_smc_get_temperature, METH_VARARGS, - "Temperature of SMC key as float"}, - {"sensors_fans", psutil_sensors_fans, METH_VARARGS, - "Return the RPM of the fan with SMC key"}, {"sensors_battery", psutil_sensors_battery, METH_VARARGS, "Return battery information."}, diff --git a/psutil/arch/osx/smc.c b/psutil/arch/osx/smc.c deleted file mode 100644 index 00de0a372..000000000 --- a/psutil/arch/osx/smc.c +++ /dev/null @@ -1,238 +0,0 @@ -/* - * Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. - * Use of this source code is governed by a BSD-style license that can be - * found in the LICENSE file. - * - * Interface to SMC API, needed in order to collect sensors stats. - */ - -#include -#include -#include - -#include "smc.h" - -UInt32 _strtoul(char *str, int size, int base) -{ - UInt32 total = 0; - int i; - - for (i = 0; i < size; i++) { - if (base == 16) - total += str[i] << (size - 1 - i) * 8; - else - total += (unsigned char) (str[i] << (size - 1 - i) * 8); - } - return total; -} - - -float _strtof(unsigned char *str, int size, int e) -{ - float total = 0; - int i; - - for (i = 0; i < size; i++) { - if (i == (size - 1)) - total += (str[i] & 0xff) >> e; - else - total += str[i] << (size - 1 - i) * (8 - e); - } - - total += (str[size-1] & 0x03) * 0.25; - - return total; -} - - -void _ultostr(char *str, UInt32 val) -{ - str[0] = '\0'; - sprintf(str, "%c%c%c%c", - (unsigned int) val >> 24, - (unsigned int) val >> 16, - (unsigned int) val >> 8, - (unsigned int) val); -} - - -kern_return_t SMCOpen(io_connect_t * pconn) -{ - kern_return_t result; - io_iterator_t iterator; - io_object_t device; - - CFMutableDictionaryRef matchingDictionary = IOServiceMatching("AppleSMC"); - result = IOServiceGetMatchingServices( - kIOMasterPortDefault, - matchingDictionary, - &iterator - ); - if (result != kIOReturnSuccess) { - return 1; - } - - device = IOIteratorNext(iterator); - IOObjectRelease(iterator); - if (device == 0) { - return 1; - } - - result = IOServiceOpen(device, mach_task_self(), 0, pconn); - IOObjectRelease(device); - if (result != kIOReturnSuccess) { - return 1; - } - - return kIOReturnSuccess; -} - - -kern_return_t SMCClose(io_connect_t conn) -{ - return IOServiceClose(conn); -} - - -kern_return_t SMCCall(io_connect_t conn, - int index, SMCKeyData_t *inputStructure, SMCKeyData_t *outputStructure) -{ - size_t structureInputSize; - size_t structureOutputSize; - - structureInputSize = sizeof(SMCKeyData_t); - structureOutputSize = sizeof(SMCKeyData_t); - -#if MAC_OS_X_VERSION_10_5 - return IOConnectCallStructMethod( - conn, - index, - inputStructure, - structureInputSize, - outputStructure, - &structureOutputSize); -#else - return IOConnectMethodStructureIStructureO( - conn, - index, - structureInputSize, - &structureOutputSize, - inputStructure, - outputStructure); -#endif -} - - -kern_return_t SMCReadKey(io_connect_t conn, UInt32Char_t key, SMCVal_t *val) { - kern_return_t result; - SMCKeyData_t inputStructure; - SMCKeyData_t outputStructure; - - memset(&inputStructure, 0, sizeof(SMCKeyData_t)); - memset(&outputStructure, 0, sizeof(SMCKeyData_t)); - memset(val, 0, sizeof(SMCVal_t)); - - inputStructure.key = _strtoul(key, 4, 16); - inputStructure.data8 = SMC_CMD_READ_KEYINFO; - - result = SMCCall( - conn, - KERNEL_INDEX_SMC, - &inputStructure, - &outputStructure - ); - if (result != kIOReturnSuccess) - return result; - - val->dataSize = outputStructure.keyInfo.dataSize; - _ultostr(val->dataType, outputStructure.keyInfo.dataType); - inputStructure.keyInfo.dataSize = val->dataSize; - inputStructure.data8 = SMC_CMD_READ_BYTES; - - result = SMCCall( - conn, - KERNEL_INDEX_SMC, - &inputStructure, - &outputStructure - ); - if (result != kIOReturnSuccess) - return result; - - memcpy(val->bytes, outputStructure.bytes, sizeof(outputStructure.bytes)); - - return kIOReturnSuccess; -} - - -double SMCGetTemperature(char *key) { - io_connect_t conn; - SMCVal_t val; - kern_return_t result; - int intValue; - - result = SMCOpen(&conn); - if (result != kIOReturnSuccess) { - return 0.0; - } - - result = SMCReadKey(conn, key, &val); - if (result == kIOReturnSuccess) { - // read succeeded - check returned value - if (val.dataSize > 0) { - if (strcmp(val.dataType, DATATYPE_SP78) == 0) { - // convert sp78 value to temperature - intValue = val.bytes[0] * 256 + (unsigned char)val.bytes[1]; - SMCClose(conn); - return intValue / 256.0; - } - } - } - // read failed - SMCClose(conn); - return 0.0; -} - - -float SMCGetFanSpeed(int fanNum) -{ - SMCVal_t val; - kern_return_t result; - UInt32Char_t key; - io_connect_t conn; - - result = SMCOpen(&conn); - if (result != kIOReturnSuccess) { - return -1; - } - - sprintf(key, SMC_KEY_FAN_SPEED, fanNum); - result = SMCReadKey(conn, key, &val); - if (result != kIOReturnSuccess) { - SMCClose(conn); - return -1; - } - SMCClose(conn); - return _strtof((unsigned char *)val.bytes, val.dataSize, 2); -} - - - -int SMCGetFanNumber(char *key) -{ - SMCVal_t val; - kern_return_t result; - io_connect_t conn; - - result = SMCOpen(&conn); - if (result != kIOReturnSuccess) { - return 0; - } - - result = SMCReadKey(conn, key, &val); - if (result != kIOReturnSuccess) { - SMCClose(conn); - return 0; - } - SMCClose(conn); - return _strtoul((char *)val.bytes, val.dataSize, 10); -} diff --git a/psutil/arch/osx/smc.h b/psutil/arch/osx/smc.h deleted file mode 100644 index acd399c2e..000000000 --- a/psutil/arch/osx/smc.h +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. - * Use of this source code is governed by a BSD-style license that can be - * found in the LICENSE file. - */ - -#ifndef __SMC_H__ -#define __SMC_H__ - -#define KERNEL_INDEX_SMC 2 - -#define SMC_CMD_READ_BYTES 5 -#define SMC_CMD_WRITE_BYTES 6 -#define SMC_CMD_READ_INDEX 8 -#define SMC_CMD_READ_KEYINFO 9 -#define SMC_CMD_READ_PLIMIT 11 -#define SMC_CMD_READ_VERS 12 - - -#define DATATYPE_FPE2 "fpe2" -#define DATATYPE_UINT8 "ui8 " -#define DATATYPE_UINT16 "ui16" -#define DATATYPE_UINT32 "ui32" -#define DATATYPE_SP78 "sp78" - -// Fans SMC key values -#define SMC_KEY_FAN_SPEED "F%dAc" -#define SMC_KEY_FAN_NUM "FNum" - -typedef struct { - char major; - char minor; - char build; - char reserved[1]; - UInt16 release; -} SMCKeyData_vers_t; - -typedef struct { - UInt16 version; - UInt16 length; - UInt32 cpuPLimit; - UInt32 gpuPLimit; - UInt32 memPLimit; -} SMCKeyData_pLimitData_t; - -typedef struct { - UInt32 dataSize; - UInt32 dataType; - char dataAttributes; -} SMCKeyData_keyInfo_t; - -typedef char SMCBytes_t[32]; - -typedef struct { - UInt32 key; - SMCKeyData_vers_t vers; - SMCKeyData_pLimitData_t pLimitData; - SMCKeyData_keyInfo_t keyInfo; - char result; - char status; - char data8; - UInt32 data32; - SMCBytes_t bytes; -} SMCKeyData_t; - -typedef char UInt32Char_t[5]; - -typedef struct { - UInt32Char_t key; - UInt32 dataSize; - UInt32Char_t dataType; - SMCBytes_t bytes; -} SMCVal_t; - -double SMCGetTemperature(char *key); -int SMCGetFanNumber(char *key); -float SMCGetFanSpeed(int fanNum); - -#endif diff --git a/psutil/tests/test_contracts.py b/psutil/tests/test_contracts.py index da1c0c856..877a5c066 100755 --- a/psutil/tests/test_contracts.py +++ b/psutil/tests/test_contracts.py @@ -118,10 +118,10 @@ def test_cpu_freq(self): def test_sensors_temperatures(self): self.assertEqual( - hasattr(psutil, "sensors_temperatures"), LINUX or MACOS) + hasattr(psutil, "sensors_temperatures"), LINUX) def test_sensors_fans(self): - self.assertEqual(hasattr(psutil, "sensors_fans"), LINUX or MACOS) + self.assertEqual(hasattr(psutil, "sensors_fans"), LINUX) def test_battery(self): self.assertEqual(hasattr(psutil, "sensors_battery"), diff --git a/setup.py b/setup.py index adc7a0e2d..c4845a8d7 100755 --- a/setup.py +++ b/setup.py @@ -152,7 +152,6 @@ def get_winver(): sources=sources + [ 'psutil/_psutil_osx.c', 'psutil/arch/osx/process_info.c', - 'psutil/arch/osx/smc.c', ], define_macros=macros, extra_link_args=[ @@ -322,6 +321,7 @@ def main(): 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: Implementation :: CPython', 'Programming Language :: Python :: Implementation :: PyPy', 'Programming Language :: Python', From da4e3598fa245fb081bba101fd9cc698c81a8033 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 14 Aug 2018 01:59:34 +0200 Subject: [PATCH 114/182] set version to 5.4.7 --- HISTORY.rst | 2 +- MANIFEST.in | 4 +--- docs/index.rst | 4 ++-- psutil/__init__.py | 2 +- 4 files changed, 5 insertions(+), 7 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index e295436ab..5ca1813aa 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,6 +1,6 @@ *Bug tracker at https://github.com/giampaolo/psutil/issues* -5.5.0 +5.4.7 ===== XXXX-XX-XX diff --git a/MANIFEST.in b/MANIFEST.in index 146cd92df..26d678fc3 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -63,8 +63,6 @@ include psutil/arch/openbsd/specific.c include psutil/arch/openbsd/specific.h include psutil/arch/osx/process_info.c include psutil/arch/osx/process_info.h -include psutil/arch/osx/smc.c -include psutil/arch/osx/smc.h include psutil/arch/solaris/environ.c include psutil/arch/solaris/environ.h include psutil/arch/solaris/v10/ifaddrs.c @@ -88,9 +86,9 @@ include psutil/tests/test_bsd.py include psutil/tests/test_connections.py include psutil/tests/test_contracts.py include psutil/tests/test_linux.py -include psutil/tests/test_osx.py include psutil/tests/test_memory_leaks.py include psutil/tests/test_misc.py +include psutil/tests/test_osx.py include psutil/tests/test_posix.py include psutil/tests/test_process.py include psutil/tests/test_sunos.py diff --git a/docs/index.rst b/docs/index.rst index c70d2f497..a9122995d 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -2135,7 +2135,7 @@ Constants Alias for :const:`MACOS` (deprecated). .. warning:: - deprecated in version 5.5.0; use :const:`MACOS` instead. + deprecated in version 5.4.7; use :const:`MACOS` instead. .. _const-procfs_path: .. data:: PROCFS_PATH @@ -2179,7 +2179,7 @@ Constants Returned by :meth:`psutil.Process.status()`. .. versionadded:: 3.4.1 STATUS_SUSPENDED (NetBSD) - .. versionadded:: 5.5.0 STATUS_PARKED (Linux) + .. versionadded:: 5.4.7 STATUS_PARKED (Linux) .. _const-conn: .. data:: CONN_ESTABLISHED diff --git a/psutil/__init__.py b/psutil/__init__.py index 291d0d16a..e1299655a 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -221,7 +221,7 @@ ] __all__.extend(_psplatform.__extra__all__) __author__ = "Giampaolo Rodola'" -__version__ = "5.4.6" +__version__ = "5.4.7" version_info = tuple([int(num) for num in __version__.split('.')]) AF_LINK = _psplatform.AF_LINK POWER_TIME_UNLIMITED = _common.POWER_TIME_UNLIMITED From feba8be76167c457f678b29a6ef4d00cc7babc82 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 14 Aug 2018 22:57:25 +0200 Subject: [PATCH 115/182] pre release --- HISTORY.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index 5ca1813aa..54ee4795f 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -3,7 +3,7 @@ 5.4.7 ===== -XXXX-XX-XX +2018-08-14 **Enhancements** From 0699c04eb20132dc6f15f69e29565c67d264f00e Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 14 Aug 2018 23:00:51 +0200 Subject: [PATCH 116/182] pre release --- docs/index.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/index.rst b/docs/index.rst index a9122995d..59f768150 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -2635,6 +2635,10 @@ take a look at the Timeline ======== +- 2018-08-14: + `5.4.7 `__ - + `what's new `__ - + `diff `__ - 2018-06-07: `5.4.6 `__ - `what's new `__ - From df027542ca60998a6d0d4e1a2f59fb720ffccfc0 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 15 Aug 2018 14:48:48 +0200 Subject: [PATCH 117/182] make test more robust --- .travis.yml | 2 +- psutil/__init__.py | 2 +- psutil/tests/test_system.py | 11 ++++++----- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 86cc73ec0..6696870f4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,7 +15,7 @@ matrix: env: PYVER=py27 - language: generic os: osx - env: PYVER=py34 + env: PYVER=py36 install: - ./.ci/travis/install.sh script: diff --git a/psutil/__init__.py b/psutil/__init__.py index e1299655a..23dc1227e 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -221,7 +221,7 @@ ] __all__.extend(_psplatform.__extra__all__) __author__ = "Giampaolo Rodola'" -__version__ = "5.4.7" +__version__ = "5.4.8" version_info = tuple([int(num) for num in __version__.split('.')]) AF_LINK = _psplatform.AF_LINK POWER_TIME_UNLIMITED = _common.POWER_TIME_UNLIMITED diff --git a/psutil/tests/test_system.py b/psutil/tests/test_system.py index f9006ce81..8b07caff6 100755 --- a/psutil/tests/test_system.py +++ b/psutil/tests/test_system.py @@ -318,11 +318,12 @@ def test_cpu_times(self): def test_cpu_times_time_increases(self): # Make sure time increases between calls. t1 = sum(psutil.cpu_times()) - time.sleep(0.1) - t2 = sum(psutil.cpu_times()) - difference = t2 - t1 - if not difference >= 0.05: - self.fail("difference %s" % difference) + stop_at = time.time() + 1 + while time.time() < stop_at: + t2 = sum(psutil.cpu_times()) + if t2 > t1: + return + self.fail("time remained the same") def test_per_cpu_times(self): # Check type, value >= 0, str(). From fc415a28340ff6f9e4c5438a5ce68a61c9a3a602 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 15 Aug 2018 14:50:00 +0200 Subject: [PATCH 118/182] remove failing test --- psutil/tests/test_process.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index 230819623..9d786d340 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -538,9 +538,6 @@ def test_threads(self): with ThreadTask(): step2 = p.threads() self.assertEqual(len(step2), len(step1) + 1) - # on Linux, first thread id is supposed to be this process - if LINUX: - self.assertEqual(step2[0].id, os.getpid()) athread = step2[0] # test named tuple self.assertEqual(athread.id, athread[0]) From 625503d2fa9b5bd8bb30dce775ac8c2e909f8c10 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 15 Aug 2018 14:51:48 +0200 Subject: [PATCH 119/182] remove failing test assertions --- psutil/tests/test_process.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index 9d786d340..80f9796ff 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -659,16 +659,10 @@ def normpath(p): def test_memory_percent(self): p = psutil.Process() - ret = p.memory_percent() - assert 0 <= ret <= 100, ret - ret = p.memory_percent(memtype='vms') - assert 0 <= ret <= 100, ret - assert 0 <= ret <= 100, ret + p.memory_percent() self.assertRaises(ValueError, p.memory_percent, memtype="?!?") if LINUX or MACOS or WINDOWS: - ret = p.memory_percent(memtype='uss') - assert 0 <= ret <= 100, ret - assert 0 <= ret <= 100, ret + p.memory_percent(memtype='uss') def test_is_running(self): sproc = get_test_subprocess() From 601b847e1d970f7c738e771b685295f366a84b3e Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 24 Aug 2018 12:41:55 +0200 Subject: [PATCH 120/182] add download badge --- .travis.yml | 2 +- README.rst | 4 ++++ psutil/_psutil_osx.c | 4 ++-- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6696870f4..86cc73ec0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,7 +15,7 @@ matrix: env: PYVER=py27 - language: generic os: osx - env: PYVER=py36 + env: PYVER=py34 install: - ./.ci/travis/install.sh script: diff --git a/README.rst b/README.rst index abd5816c6..2fa59be65 100644 --- a/README.rst +++ b/README.rst @@ -1,3 +1,7 @@ +.. image:: http://pepy.tech/badge/psutil + :target: http://pepy.tech/project/psutil + :alt: Downloads + .. image:: https://img.shields.io/travis/giampaolo/psutil/master.svg?maxAge=3600&label=Linux%20/%20macOS :target: https://travis-ci.org/giampaolo/psutil :alt: Linux tests (Travis) diff --git a/psutil/_psutil_osx.c b/psutil/_psutil_osx.c index 18dbe8931..b75c78b20 100644 --- a/psutil/_psutil_osx.c +++ b/psutil/_psutil_osx.c @@ -682,8 +682,8 @@ psutil_proc_memory_uss(PyObject *self, PyObject *args) { /* * Return system virtual memory stats. * See: - * http://opensource.apple.com/source/system_cmds/system_cmds-498.2/ - * vm_stat.tproj/vm_stat.c + * https://opensource.apple.com/source/system_cmds/system_cmds-790/ + * vm_stat.tproj/vm_stat.c.auto.html */ static PyObject * psutil_virtual_mem(PyObject *self, PyObject *args) { From 5aa81f2892f6b4c1f5cb7cfbb8d1ec552aebe416 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 24 Aug 2018 12:49:45 +0200 Subject: [PATCH 121/182] travis: add python 3.7 build --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index 86cc73ec0..e32f87c24 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,6 +9,9 @@ matrix: - python: 3.4 - python: 3.5 - python: 3.6 + - python: 3.7 + dist: xenial + sudo: true # macOS - language: generic os: osx From fe8140c23f481578689fdf0fb95e2d3e301e73c2 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 24 Aug 2018 12:53:23 +0200 Subject: [PATCH 122/182] also include PYPY (or try to :P) --- .ci/travis/install.sh | 6 +++--- .travis.yml | 5 ++++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.ci/travis/install.sh b/.ci/travis/install.sh index bb86700ea..56f6623d8 100755 --- a/.ci/travis/install.sh +++ b/.ci/travis/install.sh @@ -24,9 +24,9 @@ if [[ "$(uname -s)" == 'Darwin' ]]; then pyenv install 2.7.10 pyenv virtualenv 2.7.10 psutil ;; - py34) - pyenv install 3.4.3 - pyenv virtualenv 3.4.3 psutil + py36) + pyenv install 3.6.6 + pyenv virtualenv 3.6.6 psutil ;; esac pyenv rehash diff --git a/.travis.yml b/.travis.yml index e32f87c24..e522fe002 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,7 +18,10 @@ matrix: env: PYVER=py27 - language: generic os: osx - env: PYVER=py34 + env: PYVER=py36 + # pypy + - python: pypy + - python: pypy3 install: - ./.ci/travis/install.sh script: From 90cb9d19937e630303d5e87e49a4132260316b53 Mon Sep 17 00:00:00 2001 From: yanok Date: Wed, 29 Aug 2018 23:28:50 +0200 Subject: [PATCH 123/182] make psutil_debug() aware of PSUTIL_DEBUG (#1332) Fixes #1331 --- psutil/_psutil_common.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/psutil/_psutil_common.c b/psutil/_psutil_common.c index 49b91c826..db1a04601 100644 --- a/psutil/_psutil_common.c +++ b/psutil/_psutil_common.c @@ -84,11 +84,13 @@ psutil_set_testing(PyObject *self, PyObject *args) { void psutil_debug(const char* format, ...) { va_list argptr; - va_start(argptr, format); - fprintf(stderr, "psutil-debug> "); - vfprintf(stderr, format, argptr); - fprintf(stderr, "\n"); - va_end(argptr); + if (PSUTIL_DEBUG) { + va_start(argptr, format); + fprintf(stderr, "psutil-debug> "); + vfprintf(stderr, format, argptr); + fprintf(stderr, "\n"); + va_end(argptr); + } } From 5e90b0a7f3fccb177445a186cc4fac62cfadb510 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 29 Aug 2018 23:31:42 +0200 Subject: [PATCH 124/182] #1332 - update HISTORY --- CREDITS | 4 ++++ HISTORY.rst | 10 ++++++++++ 2 files changed, 14 insertions(+) diff --git a/CREDITS b/CREDITS index 27a4ed148..8df9090fc 100644 --- a/CREDITS +++ b/CREDITS @@ -555,3 +555,7 @@ I: 1294 N: Lawrence Ye W: https://github.com/LEAFERx I: 1321 + +N: yanok +W: https://github.com/yanok +I: 1332 diff --git a/HISTORY.rst b/HISTORY.rst index 54ee4795f..1cd61ed86 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,5 +1,15 @@ *Bug tracker at https://github.com/giampaolo/psutil/issues* +5.4.8 +===== + +XXXX-XX-XX + +**Bug fixes** + +- 1332_: psutil debug messages are erroneously printed all the time. (patch by + yanok) + 5.4.7 ===== From 39811bbb989497b04b4bebd927de6c3e806d0e70 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 26 Sep 2018 12:59:08 +0200 Subject: [PATCH 125/182] fix #1343: document Process.as_dict() attrs values --- docs/index.rst | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 59f768150..a3fca7b5f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1179,11 +1179,14 @@ Process class Utility method retrieving multiple process information as a dictionary. If *attrs* is specified it must be a list of strings reflecting available - :class:`Process` class's attribute names (e.g. ``['cpu_times', 'name']``), - else all public (read only) attributes are assumed. *ad_value* is the - value which gets assigned to a dict key in case :class:`AccessDenied` - or :class:`ZombieProcess` exception is raised when retrieving that - particular process information. + :class:`Process` class's attribute names. Here's a list of possible string + values: + ``'cmdline'``, ``'connections'``, ``'cpu_affinity'``, ``'cpu_num'``, ``'cpu_percent'``, ``'cpu_times'``, ``'create_time'``, ``'cwd'``, ``'environ'``, ``'exe'``, ``'gids'``, ``'io_counters'``, ``'ionice'``, ``'memory_full_info'``, ``'memory_info'``, ``'memory_maps'``, ``'memory_percent'``, ``'name'``, ``'nice'``, ``'num_ctx_switches'``, ``'num_fds'``, ``'num_handles'``, ``'num_threads'``, ``'open_files'``, ``'pid'``, ``'ppid'``, ``'status'``, ``'terminal'``, ``'threads'``, ``'uids'``, ``'username'```. + If *attrs* argument is not passed all public read only attributes are + assumed. + *ad_value* is the value which gets assigned to a dict key in case + :class:`AccessDenied` or :class:`ZombieProcess` exception is raised when + retrieving that particular process information. Internally, :meth:`as_dict` uses :meth:`oneshot` context manager so there's no need you use it also. From 772c675c39f02d94eba5d727f3dd2861569d8c6c Mon Sep 17 00:00:00 2001 From: "E. M. Bray" Date: Wed, 26 Sep 2018 13:07:47 +0200 Subject: [PATCH 126/182] Refactored ps() function in test_posix (#1341) * refactored ps() function that attepts to abstract away more platform-specific differences in the ps command * move open_text an open_binary tests to test_misc --- psutil/tests/test_linux.py | 8 --- psutil/tests/test_misc.py | 10 ++++ psutil/tests/test_posix.py | 105 ++++++++++++++++++++++--------------- 3 files changed, 72 insertions(+), 51 deletions(-) diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index 4e652a9d1..5e3c834ef 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -1978,14 +1978,6 @@ def test_cpu_affinity_eligible_cpus(self): @unittest.skipIf(not LINUX, "LINUX only") class TestUtils(unittest.TestCase): - def test_open_text(self): - with psutil._psplatform.open_text(__file__) as f: - self.assertEqual(f.mode, 'rt') - - def test_open_binary(self): - with psutil._psplatform.open_binary(__file__) as f: - self.assertEqual(f.mode, 'rb') - def test_readlink(self): with mock.patch("os.readlink", return_value="foo (deleted)") as m: self.assertEqual(psutil._psplatform.readlink("bar"), "foo") diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index 1d9067e77..3056abc0a 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -26,6 +26,8 @@ from psutil._common import memoize_when_activated from psutil._common import supports_ipv6 from psutil._common import wrap_numbers +from psutil._common import open_text +from psutil._common import open_binary from psutil._compat import PY3 from psutil.tests import APPVEYOR from psutil.tests import bind_socket @@ -896,6 +898,14 @@ def setUp(self): tearDown = setUp + def test_open_text(self): + with open_text(__file__) as f: + self.assertEqual(f.mode, 'rt') + + def test_open_binary(self): + with open_binary(__file__) as f: + self.assertEqual(f.mode, 'rb') + def test_safe_mkdir(self): safe_mkdir(TESTFN) assert os.path.isdir(TESTFN) diff --git a/psutil/tests/test_posix.py b/psutil/tests/test_posix.py index b80128c7f..35f73eb80 100755 --- a/psutil/tests/test_posix.py +++ b/psutil/tests/test_posix.py @@ -12,7 +12,6 @@ import os import re import subprocess -import sys import time import psutil @@ -23,7 +22,6 @@ from psutil import OPENBSD from psutil import POSIX from psutil import SUNOS -from psutil._compat import PY3 from psutil.tests import APPVEYOR from psutil.tests import get_kernel_version from psutil.tests import get_test_subprocess @@ -40,23 +38,54 @@ from psutil.tests import which -def ps(cmd): - """Expects a ps command with a -o argument and parse the result - returning only the value of interest. +def ps(fmt, pid=None): """ - if not LINUX: - cmd = cmd.replace(" --no-headers ", " ") + Wrapper for calling the ps command with a little bit of cross-platform + support for a narrow range of features. + """ + + cmd = ['ps'] + + if LINUX: + cmd.append('--no-headers') + + if pid is not None: + cmd.extend(['-p', str(pid)]) + else: + if SUNOS: + cmd.append('-A') + else: + cmd.append('ax') + if SUNOS: - cmd = cmd.replace("-o start", "-o stime") - if AIX: - cmd = cmd.replace("-o rss", "-o rssize") + fmt_map = {'command', 'comm', + 'start', 'stime'} + fmt = fmt_map.get(fmt, fmt) + + cmd.extend(['-o', fmt]) + output = sh(cmd) - if not LINUX: - output = output.split('\n')[1].strip() - try: - return int(output) - except ValueError: - return output + + if LINUX: + output = output.splitlines() + else: + output = output.splitlines()[1:] + + all_output = [] + for line in output: + line = line.strip() + + try: + line = int(line) + except ValueError: + pass + + all_output.append(line) + + if pid is None: + return all_output + else: + return all_output[0] # ps "-o" field names differ wildly between platforms. # "comm" means "only executable name" but is not available on BSD platforms. @@ -74,14 +103,14 @@ def ps_name(pid): field = "command" if SUNOS: field = "comm" - return ps("ps --no-headers -o %s -p %s" % (field, pid)).split(' ')[0] + return ps(field, pid).split()[0] def ps_args(pid): field = "command" if AIX or SUNOS: field = "args" - return ps("ps --no-headers -o %s -p %s" % (field, pid)) + return ps(field, pid) @unittest.skipIf(not POSIX, "POSIX only") @@ -99,22 +128,22 @@ def tearDownClass(cls): reap_children() def test_ppid(self): - ppid_ps = ps("ps --no-headers -o ppid -p %s" % self.pid) + ppid_ps = ps('ppid', self.pid) ppid_psutil = psutil.Process(self.pid).ppid() self.assertEqual(ppid_ps, ppid_psutil) def test_uid(self): - uid_ps = ps("ps --no-headers -o uid -p %s" % self.pid) + uid_ps = ps('uid', self.pid) uid_psutil = psutil.Process(self.pid).uids().real self.assertEqual(uid_ps, uid_psutil) def test_gid(self): - gid_ps = ps("ps --no-headers -o rgid -p %s" % self.pid) + gid_ps = ps('rgid', self.pid) gid_psutil = psutil.Process(self.pid).gids().real self.assertEqual(gid_ps, gid_psutil) def test_username(self): - username_ps = ps("ps --no-headers -o user -p %s" % self.pid) + username_ps = ps('user', self.pid) username_psutil = psutil.Process(self.pid).username() self.assertEqual(username_ps, username_psutil) @@ -133,7 +162,7 @@ def test_rss_memory(self): # give python interpreter some time to properly initialize # so that the results are the same time.sleep(0.1) - rss_ps = ps("ps --no-headers -o rss -p %s" % self.pid) + rss_ps = ps('rss', self.pid) rss_psutil = psutil.Process(self.pid).memory_info()[0] / 1024 self.assertEqual(rss_ps, rss_psutil) @@ -143,7 +172,7 @@ def test_vsz_memory(self): # give python interpreter some time to properly initialize # so that the results are the same time.sleep(0.1) - vsz_ps = ps("ps --no-headers -o vsz -p %s" % self.pid) + vsz_ps = ps('vsz', self.pid) vsz_psutil = psutil.Process(self.pid).memory_info()[1] / 1024 self.assertEqual(vsz_ps, vsz_psutil) @@ -196,7 +225,7 @@ def test_name_long_cmdline_nsp_exc(self): @unittest.skipIf(MACOS or BSD, 'ps -o start not available') def test_create_time(self): - time_ps = ps("ps --no-headers -o start -p %s" % self.pid).split(' ')[0] + time_ps = ps('start', self.pid) time_psutil = psutil.Process(self.pid).create_time() time_psutil_tstamp = datetime.datetime.fromtimestamp( time_psutil).strftime("%H:%M:%S") @@ -235,7 +264,7 @@ def test_cmdline(self): @unittest.skipIf(SUNOS, "not reliable on SUNOS") @unittest.skipIf(AIX, "not reliable on AIX") def test_nice(self): - ps_nice = ps("ps --no-headers -o nice -p %s" % self.pid) + ps_nice = ps('nice', self.pid) psutil_nice = psutil.Process().nice() self.assertEqual(ps_nice, psutil_nice) @@ -289,22 +318,7 @@ class TestSystemAPIs(unittest.TestCase): def test_pids(self): # Note: this test might fail if the OS is starting/killing # other processes in the meantime - if SUNOS or AIX: - cmd = ["ps", "-A", "-o", "pid"] - else: - cmd = ["ps", "ax", "-o", "pid"] - p = get_test_subprocess(cmd, stdout=subprocess.PIPE) - output = p.communicate()[0].strip() - assert p.poll() == 0 - if PY3: - output = str(output, sys.stdout.encoding) - pids_ps = [] - for line in output.split('\n')[1:]: - if line: - pid = int(line.split()[0].strip()) - pids_ps.append(pid) - # remove ps subprocess pid which is supposed to be dead in meantime - pids_ps.remove(p.pid) + pids_ps = ps("pid") pids_psutil = psutil.pids() pids_ps.sort() pids_psutil.sort() @@ -312,7 +326,12 @@ def test_pids(self): # on MACOS and OPENBSD ps doesn't show pid 0 if MACOS or OPENBSD and 0 not in pids_ps: pids_ps.insert(0, 0) - self.assertEqual(pids_ps, pids_psutil) + + # There will often be one more process in pids_ps for ps itself + if len(pids_ps) - len(pids_psutil) > 1: + difference = [x for x in pids_psutil if x not in pids_ps] + \ + [x for x in pids_ps if x not in pids_psutil] + self.fail("difference: " + str(difference)) # for some reason ifconfig -a does not report all interfaces # returned by psutil From ab9a381f5cd9b33b244a4883d6ba55208c049dcf Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 26 Sep 2018 13:13:36 +0200 Subject: [PATCH 127/182] #1341: move open() utilities/wrappers in _common.py --- psutil/_common.py | 18 ++++++++++++++++++ psutil/_pslinux.py | 20 ++------------------ psutil/tests/test_linux.py | 22 +++++++++++----------- 3 files changed, 31 insertions(+), 29 deletions(-) diff --git a/psutil/_common.py b/psutil/_common.py index 2cc3939a1..bee957927 100644 --- a/psutil/_common.py +++ b/psutil/_common.py @@ -576,3 +576,21 @@ def wrap_numbers(input_dict, name): _wn = _WrapNumbers() wrap_numbers.cache_clear = _wn.cache_clear wrap_numbers.cache_info = _wn.cache_info + + +def open_binary(fname, **kwargs): + return open(fname, "rb", **kwargs) + + +def open_text(fname, **kwargs): + """On Python 3 opens a file in text mode by using fs encoding and + a proper en/decoding errors handler. + On Python 2 this is just an alias for open(name, 'rt'). + """ + if PY3: + # See: + # https://github.com/giampaolo/psutil/issues/675 + # https://github.com/giampaolo/psutil/pull/733 + kwargs.setdefault('encoding', ENCODING) + kwargs.setdefault('errors', ENCODING_ERRS) + return open(fname, "rt", **kwargs) diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index df624de32..1575dad9d 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -33,6 +33,8 @@ from ._common import NIC_DUPLEX_FULL from ._common import NIC_DUPLEX_HALF from ._common import NIC_DUPLEX_UNKNOWN +from ._common import open_binary +from ._common import open_text from ._common import parse_environ_block from ._common import path_exists_strict from ._common import supports_ipv6 @@ -201,24 +203,6 @@ class IOPriority(enum.IntEnum): # ===================================================================== -def open_binary(fname, **kwargs): - return open(fname, "rb", **kwargs) - - -def open_text(fname, **kwargs): - """On Python 3 opens a file in text mode by using fs encoding and - a proper en/decoding errors handler. - On Python 2 this is just an alias for open(name, 'rt'). - """ - if PY3: - # See: - # https://github.com/giampaolo/psutil/issues/675 - # https://github.com/giampaolo/psutil/pull/733 - kwargs.setdefault('encoding', ENCODING) - kwargs.setdefault('errors', ENCODING_ERRS) - return open(fname, "rt", **kwargs) - - if PY3: def decode(s): return s.decode(encoding=ENCODING, errors=ENCODING_ERRS) diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index 5e3c834ef..a0527851f 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -528,7 +528,7 @@ def test_free(self): free_value, psutil_value, delta=MEMORY_TOLERANCE) def test_missing_sin_sout(self): - with mock.patch('psutil._pslinux.open', create=True) as m: + with mock.patch('psutil._common.open', create=True) as m: with warnings.catch_warnings(record=True) as ws: warnings.simplefilter("always") ret = psutil.swap_memory() @@ -651,7 +651,7 @@ def test_cpu_count_logical_mocked(self): # Let's have open() return emtpy data and make sure None is # returned ('cause we mimick os.cpu_count()). - with mock.patch('psutil._pslinux.open', create=True) as m: + with mock.patch('psutil._common.open', create=True) as m: self.assertIsNone(psutil._pslinux.cpu_count_logical()) self.assertEqual(m.call_count, 2) # /proc/stat should be the last one @@ -662,7 +662,7 @@ def test_cpu_count_logical_mocked(self): with open('/proc/cpuinfo', 'rb') as f: cpuinfo_data = f.read() fake_file = io.BytesIO(cpuinfo_data) - with mock.patch('psutil._pslinux.open', + with mock.patch('psutil._common.open', return_value=fake_file, create=True) as m: self.assertEqual(psutil._pslinux.cpu_count_logical(), original) @@ -675,7 +675,7 @@ def test_cpu_count_logical_mocked(self): def test_cpu_count_physical_mocked(self): # Have open() return emtpy data and make sure None is returned # ('cause we want to mimick os.cpu_count()) - with mock.patch('psutil._pslinux.open', create=True) as m: + with mock.patch('psutil._common.open', create=True) as m: self.assertIsNone(psutil._pslinux.cpu_count_physical()) assert m.called @@ -971,7 +971,7 @@ def test_disk_partitions_mocked(self): else: # No ZFS partitions on this system. Let's fake one. fake_file = io.StringIO(u("nodev\tzfs\n")) - with mock.patch('psutil._pslinux.open', + with mock.patch('psutil._common.open', return_value=fake_file, create=True) as m1: with mock.patch( 'psutil._pslinux.cext.disk_partitions', @@ -1232,7 +1232,7 @@ def test_cpu_steal_decrease(self): self.assertNotEqual(cpu_times_percent.user, 0) def test_boot_time_mocked(self): - with mock.patch('psutil._pslinux.open', create=True) as m: + with mock.patch('psutil._common.open', create=True) as m: self.assertRaises( RuntimeError, psutil._pslinux.boot_time) @@ -1679,7 +1679,7 @@ def test_terminal_mocked(self): # TODO: re-enable this test. # def test_num_ctx_switches_mocked(self): - # with mock.patch('psutil._pslinux.open', create=True) as m: + # with mock.patch('psutil._common.open', create=True) as m: # self.assertRaises( # NotImplementedError, # psutil._pslinux.Process(os.getpid()).num_ctx_switches) @@ -1689,12 +1689,12 @@ def test_cmdline_mocked(self): # see: https://github.com/giampaolo/psutil/issues/639 p = psutil.Process() fake_file = io.StringIO(u('foo\x00bar\x00')) - with mock.patch('psutil._pslinux.open', + with mock.patch('psutil._common.open', return_value=fake_file, create=True) as m: self.assertEqual(p.cmdline(), ['foo', 'bar']) assert m.called fake_file = io.StringIO(u('foo\x00bar\x00\x00')) - with mock.patch('psutil._pslinux.open', + with mock.patch('psutil._common.open', return_value=fake_file, create=True) as m: self.assertEqual(p.cmdline(), ['foo', 'bar', '']) assert m.called @@ -1703,12 +1703,12 @@ def test_cmdline_spaces_mocked(self): # see: https://github.com/giampaolo/psutil/issues/1179 p = psutil.Process() fake_file = io.StringIO(u('foo bar ')) - with mock.patch('psutil._pslinux.open', + with mock.patch('psutil._common.open', return_value=fake_file, create=True) as m: self.assertEqual(p.cmdline(), ['foo', 'bar']) assert m.called fake_file = io.StringIO(u('foo bar ')) - with mock.patch('psutil._pslinux.open', + with mock.patch('psutil._common.open', return_value=fake_file, create=True) as m: self.assertEqual(p.cmdline(), ['foo', 'bar', '']) assert m.called From 7be5d9c6a1d829c60855edec6543039db6316631 Mon Sep 17 00:00:00 2001 From: Jon Dufresne Date: Wed, 26 Sep 2018 06:14:50 -0700 Subject: [PATCH 128/182] Correct capitalization of PyPI (#1337) As spelled on https://pypi.org/. --- HISTORY.rst | 4 ++-- INSTALL.rst | 2 +- Makefile | 4 ++-- docs/index.rst | 2 +- make.bat | 2 +- scripts/internal/winmake.py | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 1cd61ed86..0b6f38522 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -213,7 +213,7 @@ XXXX-XX-XX **Compatibility notes** -- 1120_: .exe files for Windows are no longer uploaded on PYPI as per PEP-527; +- 1120_: .exe files for Windows are no longer uploaded on PyPI as per PEP-527; only wheels are provided. 5.3.0 @@ -1227,7 +1227,7 @@ DeprecationWarning. **Enhancements** -- 410_: host tar.gz and windows binary files are on PYPI. +- 410_: host tar.gz and windows binary files are on PyPI. - 412_: [Linux] get/set process resource limits. - 415_: [Windows] Process.get_children() is an order of magnitude faster. - 426_: [Windows] Process.name is an order of magnitude faster. diff --git a/INSTALL.rst b/INSTALL.rst index 6644fb37b..aaa1b87f0 100644 --- a/INSTALL.rst +++ b/INSTALL.rst @@ -76,7 +76,7 @@ Windows The easiest way to install psutil on Windows is to just use the pre-compiled exe/wheel installers hosted on -`PYPI `__ via pip: +`PyPI `__ via pip: .. code-block:: bat diff --git a/Makefile b/Makefile index 51047383f..25928e0aa 100644 --- a/Makefile +++ b/Makefile @@ -210,7 +210,7 @@ upload-src: ## Upload source tarball on https://pypi.org/project/psutil/ ${MAKE} sdist $(PYTHON) setup.py sdist upload -upload-win-wheels: ## Upload wheels in dist/* directory on PYPI. +upload-win-wheels: ## Upload wheels in dist/* directory on PyPI. $(PYTHON) -m twine upload dist/*.whl # --- others @@ -233,7 +233,7 @@ pre-release: ## Check if we're ready to produce a new release. release: ## Create a release (down/uploads tar.gz, wheels, git tag release). ${MAKE} pre-release - $(PYTHON) -m twine upload dist/* # upload tar.gz and Windows wheels on PYPI + $(PYTHON) -m twine upload dist/* # upload tar.gz and Windows wheels on PyPI ${MAKE} git-tag-release print-announce: ## Print announce of new release. diff --git a/docs/index.rst b/docs/index.rst index a3fca7b5f..7763b6de0 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -53,7 +53,7 @@ The easiest way to install psutil is via ``pip``:: On UNIX this requires a C compiler (e.g. gcc) installed. On Windows pip will automatically retrieve a pre-compiled wheel version from -`PYPI repository `__. +`PyPI repository `__. Alternatively, see more detailed `install `_ instructions. diff --git a/make.bat b/make.bat index c7c319026..43000535d 100644 --- a/make.bat +++ b/make.bat @@ -26,7 +26,7 @@ if "%TSCRIPT%" == "" ( set TSCRIPT=psutil\tests\__main__.py ) -rem Needed to locate the .pypirc file and upload exes on PYPI. +rem Needed to locate the .pypirc file and upload exes on PyPI. set HOME=%USERPROFILE% %PYTHON% scripts\internal\winmake.py %1 %2 %3 %4 %5 %6 diff --git a/scripts/internal/winmake.py b/scripts/internal/winmake.py index ffdfc291e..c4722a029 100755 --- a/scripts/internal/winmake.py +++ b/scripts/internal/winmake.py @@ -229,7 +229,7 @@ def wheel(): @cmd def upload_wheels(): - """Upload wheel files on PYPI.""" + """Upload wheel files on PyPI.""" build() sh("%s -m twine upload dist/*.whl" % PYTHON) From 1d7516c10cc89c60b8b5607ff9656d2f817b22b1 Mon Sep 17 00:00:00 2001 From: sylvainduchesne <35115147+sylvainduchesne@users.noreply.github.com> Date: Wed, 26 Sep 2018 14:35:12 -0400 Subject: [PATCH 129/182] Fix random 0xC0000001 errors when querying for Connections (#1335) Fix random 0xC0000001 errors when querying for Connections (#1335) --- CREDITS | 2 +- HISTORY.rst | 4 +++- psutil/_psutil_windows.c | 14 ++++++-------- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/CREDITS b/CREDITS index 8df9090fc..bfb686270 100644 --- a/CREDITS +++ b/CREDITS @@ -548,7 +548,7 @@ N: Alex Manuskin W: https://github.com/amanusk I: 1284 -N: sylvainduchesne +N: Sylvain Duchesne W: https://github.com/sylvainduchesne I: 1294 diff --git a/HISTORY.rst b/HISTORY.rst index 0b6f38522..0c79a30a6 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -9,7 +9,9 @@ XXXX-XX-XX - 1332_: psutil debug messages are erroneously printed all the time. (patch by yanok) - +- 1294_: [Windows] psutil.Process().connections() may sometimes fail with + intermittent 0xC0000001. (patch by Sylvain Duchesne) + 5.4.7 ===== diff --git a/psutil/_psutil_windows.c b/psutil/_psutil_windows.c index 1d44dd0ba..4acea3600 100644 --- a/psutil/_psutil_windows.c +++ b/psutil/_psutil_windows.c @@ -1530,15 +1530,14 @@ static DWORD __GetExtendedTcpTable(_GetExtendedTcpTable call, // that the size of the table increases between the moment where we // query the size and the moment where we query the data. Therefore, it's // important to call this in a loop to retry if that happens. - // - // Also, since we may loop a theoretically unbounded number of times here, - // release the GIL while we're doing this. + // See https://github.com/giampaolo/psutil/pull/1335 concerning 0xC0000001 error + // and https://github.com/giampaolo/psutil/issues/1294 DWORD error = ERROR_INSUFFICIENT_BUFFER; *size = 0; *data = NULL; error = call(NULL, size, FALSE, address_family, TCP_TABLE_OWNER_PID_ALL, 0); - while (error == ERROR_INSUFFICIENT_BUFFER) + while (error == ERROR_INSUFFICIENT_BUFFER || error == 0xC0000001) { *data = malloc(*size); if (*data == NULL) { @@ -1569,15 +1568,14 @@ static DWORD __GetExtendedUdpTable(_GetExtendedUdpTable call, // that the size of the table increases between the moment where we // query the size and the moment where we query the data. Therefore, it's // important to call this in a loop to retry if that happens. - // - // Also, since we may loop a theoretically unbounded number of times here, - // release the GIL while we're doing this. + // See https://github.com/giampaolo/psutil/pull/1335 concerning 0xC0000001 error + // and https://github.com/giampaolo/psutil/issues/1294 DWORD error = ERROR_INSUFFICIENT_BUFFER; *size = 0; *data = NULL; error = call(NULL, size, FALSE, address_family, UDP_TABLE_OWNER_PID, 0); - while (error == ERROR_INSUFFICIENT_BUFFER) + while (error == ERROR_INSUFFICIENT_BUFFER || error == 0xC0000001) { *data = malloc(*size); if (*data == NULL) { From 1a7512f337a61733c0523b4199aa392b48a22ad6 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 28 Sep 2018 18:45:46 +0200 Subject: [PATCH 130/182] use memory tolerance in occasionally failing test --- CREDITS | 2 +- HISTORY.rst | 6 +++--- psutil/tests/test_linux.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CREDITS b/CREDITS index bfb686270..a7e2dcbef 100644 --- a/CREDITS +++ b/CREDITS @@ -556,6 +556,6 @@ N: Lawrence Ye W: https://github.com/LEAFERx I: 1321 -N: yanok +N: Ilya Yanok W: https://github.com/yanok I: 1332 diff --git a/HISTORY.rst b/HISTORY.rst index 0c79a30a6..b1226489f 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -7,11 +7,11 @@ XXXX-XX-XX **Bug fixes** -- 1332_: psutil debug messages are erroneously printed all the time. (patch by - yanok) - 1294_: [Windows] psutil.Process().connections() may sometimes fail with intermittent 0xC0000001. (patch by Sylvain Duchesne) - +- 1332_: [OSX] psutil debug messages are erroneously printed all the time. + (patch by Ilya Yanok) + 5.4.7 ===== diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index a0527851f..a8e7a5e34 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -575,7 +575,7 @@ def test_meminfo_against_sysinfo(self): total *= unit_multiplier free *= unit_multiplier self.assertEqual(swap.total, total) - self.assertEqual(swap.free, free) + self.assertEqual(swap.free, free, delta=MEMORY_TOLERANCE) def test_emulate_meminfo_has_no_metrics(self): # Emulate a case where /proc/meminfo provides no swap metrics From cc888778bf49c6472b22ed7b5541e0c6e294c427 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 28 Sep 2018 19:09:56 +0200 Subject: [PATCH 131/182] catch UnicodeEncodeError on print() --- scripts/pmap.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/scripts/pmap.py b/scripts/pmap.py index a509bd733..988b75072 100755 --- a/scripts/pmap.py +++ b/scripts/pmap.py @@ -35,6 +35,13 @@ import psutil +def safe_print(s): + try: + print(s) + except UnicodeEncodeError: + print(s.encode('ascii', 'ignore').decode()) + + def main(): if len(sys.argv) != 2: sys.exit('usage: pmap ') @@ -45,7 +52,7 @@ def main(): total_rss = 0 for m in p.memory_maps(grouped=False): total_rss += m.rss - print(templ % ( + safe_print(templ % ( m.addr.split('-')[0].zfill(16), str(m.rss / 1024) + 'K', m.perms, From da17f5e3d0b5fc7d3812fc1556426430a8464624 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 28 Sep 2018 19:13:02 +0200 Subject: [PATCH 132/182] Fix decoding error in tests https://travis-ci.org/giampaolo/psutil/jobs/434570177 --- psutil/tests/test_process.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index 80f9796ff..a4adf367f 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -31,6 +31,7 @@ from psutil import POSIX from psutil import SUNOS from psutil import WINDOWS +from psutil._common import open_text from psutil._compat import long from psutil._compat import PY3 from psutil.tests import APPVEYOR @@ -626,7 +627,7 @@ def test_memory_maps(self): raise else: # https://github.com/giampaolo/psutil/issues/759 - with open('/proc/self/smaps') as f: + with open_text('/proc/self/smaps') as f: data = f.read() if "%s (deleted)" % nt.path not in data: raise From 13ef73cd5c41155b6cb6cd5b51cd5d6c100b4d47 Mon Sep 17 00:00:00 2001 From: Alex Manuskin Date: Mon, 1 Oct 2018 15:50:04 +0300 Subject: [PATCH 133/182] Add parsing for /sys/class/thermal (#1345) Add parsing for /sys/class/thermal --- psutil/_pslinux.py | 44 ++++++++++++++++++++++++++++++++++++++ psutil/tests/test_linux.py | 40 +++++++++++++++++++++++++++++++++- 2 files changed, 83 insertions(+), 1 deletion(-) diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index 1575dad9d..a69b1d664 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -1213,6 +1213,50 @@ def sensors_temperatures(): ret[unit_name].append((label, current, high, critical)) + # Indication that no sensors were detected in /sys/class/hwmon/ + if not basenames: + basenames = glob.glob('/sys/class/thermal/thermal_zone*') + basenames = sorted(set(basenames)) + + for base in basenames: + try: + path = os.path.join(base, 'temp') + current = float(cat(path)) / 1000.0 + path = os.path.join(base, 'type') + unit_name = cat(path, binary=False) + except (IOError, OSError, ValueError) as err: + warnings.warn("ignoring %r for file %r" % (err, path), + RuntimeWarning) + continue + + trip_paths = glob.glob(base + '/trip_point*') + trip_points = set(['_'.join( + os.path.basename(p).split('_')[0:3]) for p in trip_paths]) + critical = None + high = None + for trip_point in trip_points: + path = os.path.join(base, trip_point + "_type") + trip_type = cat(path, fallback='', binary=False) + if trip_type == 'critical': + critical = cat(os.path.join(base, trip_point + "_temp"), + fallback=None) + elif trip_type == 'high': + high = cat(os.path.join(base, trip_point + "_temp"), + fallback=None) + + if high is not None: + try: + high = float(high) / 1000.0 + except ValueError: + high = None + if critical is not None: + try: + critical = float(critical) / 1000.0 + except ValueError: + critical = None + + ret[unit_name].append(('', current, high, critical)) + return ret diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index a0527851f..c565f6c2f 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -1466,6 +1466,8 @@ def test_emulate_eio_error(self): def open_mock(name, *args, **kwargs): if name.endswith("_input"): raise OSError(errno.EIO, "") + elif name.endswith("temp"): + raise OSError(errno.EIO, "") else: return orig_open(name, *args, **kwargs) @@ -1477,7 +1479,7 @@ def open_mock(name, *args, **kwargs): assert m.called self.assertIn("ignoring", str(ws[0].message)) - def test_emulate_data(self): + def test_emulate_class_hwmon(self): def open_mock(name, *args, **kwargs): if name.endswith('/name'): return io.StringIO(u("name")) @@ -1495,6 +1497,7 @@ def open_mock(name, *args, **kwargs): orig_open = open patch_point = 'builtins.open' if PY3 else '__builtin__.open' with mock.patch(patch_point, side_effect=open_mock): + # Test case with /sys/class/hwmon with mock.patch('glob.glob', return_value=['/sys/class/hwmon/hwmon0/temp1']): temp = psutil.sensors_temperatures()['name'][0] @@ -1503,6 +1506,41 @@ def open_mock(name, *args, **kwargs): self.assertEqual(temp.high, 40.0) self.assertEqual(temp.critical, 50.0) + def test_emulate_class_thermal(self): + def open_mock(name, *args, **kwargs): + if name.endswith('0_temp'): + return io.BytesIO(b"50000") + elif name.endswith('temp'): + return io.BytesIO(b"30000") + elif name.endswith('0_type'): + return io.StringIO(u("critical")) + elif name.endswith('type'): + return io.StringIO(u("name")) + else: + return orig_open(name, *args, **kwargs) + + def glob_mock(path): + if path == '/sys/class/hwmon/hwmon*/temp*_*': + return [] + elif path == '/sys/class/hwmon/hwmon*/device/temp*_*': + return [] + elif path == '/sys/class/thermal/thermal_zone*': + return ['/sys/class/thermal/thermal_zone0'] + elif path == '/sys/class/thermal/thermal_zone0/trip_point*': + return ['/sys/class/thermal/thermal_zone1/trip_point_0_type', + '/sys/class/thermal/thermal_zone1/trip_point_0_temp'] + + orig_open = open + patch_point = 'builtins.open' if PY3 else '__builtin__.open' + with mock.patch(patch_point, side_effect=open_mock): + + with mock.patch('glob.glob', create=True, side_effect=glob_mock): + temp = psutil.sensors_temperatures()['name'][0] + self.assertEqual(temp.label, '') + self.assertEqual(temp.current, 30.0) + self.assertEqual(temp.high, 50.0) + self.assertEqual(temp.critical, 50.0) + @unittest.skipIf(not LINUX, "LINUX only") class TestSensorsFans(unittest.TestCase): From 33b7c7a7574c288c17265661d96e9f557f168aaa Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 1 Oct 2018 14:52:50 +0200 Subject: [PATCH 134/182] #1284, #1345 - give credits to @amanusk --- CREDITS | 2 +- HISTORY.rst | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CREDITS b/CREDITS index a7e2dcbef..4affa0f9e 100644 --- a/CREDITS +++ b/CREDITS @@ -546,7 +546,7 @@ I: 1278 N: Alex Manuskin W: https://github.com/amanusk -I: 1284 +I: 1284, 1345 N: Sylvain Duchesne W: https://github.com/sylvainduchesne diff --git a/HISTORY.rst b/HISTORY.rst index b1226489f..feb42ad4d 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -5,6 +5,12 @@ XXXX-XX-XX +**Enhancements** + +- 1310_: [Linux] psutil.sensors_temperatures() now parses /sys/class/thermal + in case /sys/class/hwmon fs is not available (e.g. Raspberry Pi). (patch + by Alex Manuskin) + **Bug fixes** - 1294_: [Windows] psutil.Process().connections() may sometimes fail with From a7900080aed852598776cd3b6c8408c117160e8d Mon Sep 17 00:00:00 2001 From: alxchk Date: Thu, 11 Oct 2018 13:15:08 +0300 Subject: [PATCH 135/182] Fix https://github.com/giampaolo/psutil/issues/1346 (#1347) --- psutil/_psutil_sunos.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/psutil/_psutil_sunos.c b/psutil/_psutil_sunos.c index 99069f56a..0717f1950 100644 --- a/psutil/_psutil_sunos.c +++ b/psutil/_psutil_sunos.c @@ -1173,7 +1173,7 @@ psutil_net_connections(PyObject *self, PyObject *args) { mibhdr.len = 0; #endif memcpy(buf, &tor, sizeof tor); - memcpy(buf + sizeof tor, &mibhdr, sizeof mibhdr); + memcpy(buf + tor.OPT_offset, &mibhdr, sizeof mibhdr); ctlbuf.buf = buf; ctlbuf.len = tor.OPT_offset + tor.OPT_length; @@ -1213,6 +1213,9 @@ psutil_net_connections(PyObject *self, PyObject *args) { goto error; } + memset(&mibhdr, 0x0, sizeof(mibhdr)); + memcpy(&mibhdr, buf + toa.OPT_offset, toa.OPT_length); + databuf.maxlen = mibhdr.len; databuf.len = 0; databuf.buf = (char *)malloc((int)mibhdr.len); From 15b00d8a57630adc4ad31bd28c455382d00f2d36 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 11 Oct 2018 12:17:41 +0200 Subject: [PATCH 136/182] give credits to @alxchk for #1346 (sunOS) --- CREDITS | 2 +- HISTORY.rst | 2 ++ psutil/tests/test_linux.py | 3 +-- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CREDITS b/CREDITS index 4affa0f9e..fea9143ed 100644 --- a/CREDITS +++ b/CREDITS @@ -486,7 +486,7 @@ I: 1042, 1079 N: Oleksii Shevchuk W: https://github.com/alxchk -I: 1077, 1093, 1091, 1220 +I: 1077, 1093, 1091, 1220, 1346 N: Prodesire W: https://github.com/Prodesire diff --git a/HISTORY.rst b/HISTORY.rst index feb42ad4d..3a22133ae 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -17,6 +17,8 @@ XXXX-XX-XX intermittent 0xC0000001. (patch by Sylvain Duchesne) - 1332_: [OSX] psutil debug messages are erroneously printed all the time. (patch by Ilya Yanok) +- 1346_: [SunOS] net_connections() returns an empty list. (patch by Oleksii + Shevchuk) 5.4.7 ===== diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index 299c110fe..c162666c2 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -575,7 +575,7 @@ def test_meminfo_against_sysinfo(self): total *= unit_multiplier free *= unit_multiplier self.assertEqual(swap.total, total) - self.assertEqual(swap.free, free, delta=MEMORY_TOLERANCE) + self.assertAlmostEqual(swap.free, free, delta=MEMORY_TOLERANCE) def test_emulate_meminfo_has_no_metrics(self): # Emulate a case where /proc/meminfo provides no swap metrics @@ -1533,7 +1533,6 @@ def glob_mock(path): orig_open = open patch_point = 'builtins.open' if PY3 else '__builtin__.open' with mock.patch(patch_point, side_effect=open_mock): - with mock.patch('glob.glob', create=True, side_effect=glob_mock): temp = psutil.sensors_temperatures()['name'][0] self.assertEqual(temp.label, '') From 659f3a36626e16405eee030b695136884ea4a220 Mon Sep 17 00:00:00 2001 From: Jaime Fullaondo Date: Thu, 11 Oct 2018 06:25:03 -0400 Subject: [PATCH 137/182] [aix] improve compilation on AIX, better support for gcc/g++ + fix cpu metrics (#1320) --- psutil/_psutil_aix.c | 25 +++++++++++++++++++++---- psutil/_psutil_posix.c | 8 ++++++++ 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/psutil/_psutil_aix.c b/psutil/_psutil_aix.c index 916254d5a..898da6b26 100644 --- a/psutil/_psutil_aix.c +++ b/psutil/_psutil_aix.c @@ -46,6 +46,7 @@ #include #include #include +#include #include "arch/aix/ifaddrs.h" #include "arch/aix/net_connections.h" @@ -617,6 +618,7 @@ psutil_boot_time(PyObject *self, PyObject *args) { static PyObject * psutil_per_cpu_times(PyObject *self, PyObject *args) { int ncpu, rc, i; + long ticks; perfstat_cpu_t *cpu = NULL; perfstat_id_t id; PyObject *py_retlist = PyList_New(0); @@ -625,6 +627,13 @@ psutil_per_cpu_times(PyObject *self, PyObject *args) { if (py_retlist == NULL) return NULL; + /* get the number of ticks per second */ + ticks = sysconf(_SC_CLK_TCK); + if (ticks < 0) { + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + /* get the number of cpus in ncpu */ ncpu = perfstat_cpu(NULL, NULL, sizeof(perfstat_cpu_t), 0); if (ncpu <= 0){ @@ -650,10 +659,10 @@ psutil_per_cpu_times(PyObject *self, PyObject *args) { for (i = 0; i < ncpu; i++) { py_cputime = Py_BuildValue( "(dddd)", - (double)cpu[i].user, - (double)cpu[i].sys, - (double)cpu[i].idle, - (double)cpu[i].wait); + (double)cpu[i].user / ticks, + (double)cpu[i].sys / ticks, + (double)cpu[i].idle / ticks, + (double)cpu[i].wait / ticks); if (!py_cputime) goto error; if (PyList_Append(py_retlist, py_cputime)) @@ -916,6 +925,10 @@ struct module_state { #define GETSTATE(m) (&_state) #endif +#ifdef __cplusplus +extern "C" { +#endif + #if PY_MAJOR_VERSION >= 3 static int @@ -986,3 +999,7 @@ void init_psutil_aix(void) return module; #endif } + +#ifdef __cplusplus +} +#endif diff --git a/psutil/_psutil_posix.c b/psutil/_psutil_posix.c index c851abbc1..d9a8f6d1d 100644 --- a/psutil/_psutil_posix.c +++ b/psutil/_psutil_posix.c @@ -655,6 +655,10 @@ struct module_state { #define GETSTATE(m) (&_state) #endif +#ifdef __cplusplus +extern "C" { +#endif + #if PY_MAJOR_VERSION >= 3 static int @@ -708,3 +712,7 @@ void init_psutil_posix(void) return module; #endif } + +#ifdef __cplusplus +} +#endif From 20e65b39ff0c601fad90775612c19aedb9b55713 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 11 Oct 2018 12:26:00 +0200 Subject: [PATCH 138/182] give CREDITS for #1320 to @truthbk --- CREDITS | 4 ++++ HISTORY.rst | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/CREDITS b/CREDITS index fea9143ed..b3c8164c9 100644 --- a/CREDITS +++ b/CREDITS @@ -559,3 +559,7 @@ I: 1321 N: Ilya Yanok W: https://github.com/yanok I: 1332 + +N: Jaime Fullaondo +W: https://github.com/truthbk +I: 1320 diff --git a/HISTORY.rst b/HISTORY.rst index 3a22133ae..a03b5ad25 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -10,11 +10,15 @@ XXXX-XX-XX - 1310_: [Linux] psutil.sensors_temperatures() now parses /sys/class/thermal in case /sys/class/hwmon fs is not available (e.g. Raspberry Pi). (patch by Alex Manuskin) +- 1320_: [Posix] better compilation support when using g++ instead of gcc. + (patch by Jaime Fullaondo) **Bug fixes** - 1294_: [Windows] psutil.Process().connections() may sometimes fail with intermittent 0xC0000001. (patch by Sylvain Duchesne) +- 1320_: [AIX] system CPU times (psutil.cpu_times()) were being reported with + ticks unit as opposed to seconds. (patch by Jaime Fullaondo) - 1332_: [OSX] psutil debug messages are erroneously printed all the time. (patch by Ilya Yanok) - 1346_: [SunOS] net_connections() returns an empty list. (patch by Oleksii From 1a0520d225053a818850b04e9b70ea8d3cce33a9 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 12 Oct 2018 17:48:15 +0200 Subject: [PATCH 139/182] fix different travis failures --- .travis.yml | 1 - psutil/tests/__init__.py | 12 +++--------- psutil/tests/test_aix.py | 4 ++-- psutil/tests/test_contracts.py | 3 ++- psutil/tests/test_posix.py | 3 +++ psutil/tests/test_process.py | 3 +++ psutil/tests/test_unicode.py | 3 +++ 7 files changed, 16 insertions(+), 13 deletions(-) diff --git a/.travis.yml b/.travis.yml index e522fe002..2067f8d2f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,6 @@ cache: pip matrix: include: # Linux - - python: 2.6 - python: 2.7 - python: 3.4 - python: 3.5 diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index a483ecaad..437588a6f 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -117,6 +117,7 @@ # whether we're running this test suite on Appveyor for Windows # (http://www.appveyor.com/) APPVEYOR = bool(os.environ.get('APPVEYOR')) +PYPY = '__pypy__' in sys.builtin_module_names # --- configurable defaults @@ -215,13 +216,8 @@ def attempt(exe): _testfiles_created = set() -def logstderr(s): - print(s, file=sys.stderr) - - @atexit.register def cleanup_test_files(): - logstderr("executing cleanup_test_files() atexit function") DEVNULL.close() for name in os.listdir(u('.')): if isinstance(name, unicode): @@ -229,13 +225,11 @@ def cleanup_test_files(): else: prefix = TESTFILE_PREFIX if name.startswith(prefix): - logstderr("removing temporary test file %r" % name) try: safe_rmpath(name) except Exception: traceback.print_exc() for path in _testfiles_created: - logstderr("removing temporary test file %r" % path) try: safe_rmpath(path) except Exception: @@ -245,7 +239,6 @@ def cleanup_test_files(): # this is executed first @atexit.register def cleanup_test_procs(): - logstderr("executing cleanup_test_procs() atexit function") reap_children(recursive=True) @@ -1192,11 +1185,12 @@ def copyload_shared_lib(dst_prefix=TESTFILE_PREFIX): by this process, copies it in another location and loads it in memory via ctypes. Return the new absolutized path. """ + exe = 'pypy' if PYPY else 'python' ext = ".so" dst = tempfile.mktemp(prefix=dst_prefix, suffix=ext) libs = [x.path for x in psutil.Process().memory_maps() if os.path.splitext(x.path)[1] == ext and - 'python' in x.path.lower()] + exe in x.path.lower()] src = random.choice(libs) shutil.copyfile(src, dst) try: diff --git a/psutil/tests/test_aix.py b/psutil/tests/test_aix.py index 7a8a4c334..0b29215f0 100755 --- a/psutil/tests/test_aix.py +++ b/psutil/tests/test_aix.py @@ -22,9 +22,9 @@ class AIXSpecificTestCase(unittest.TestCase): def test_virtual_memory(self): out = sh('/usr/bin/svmon -O unit=KB') - re_pattern = "memory\s*" + re_pattern = r"memory\s*" for field in ("size inuse free pin virtual available mmode").split(): - re_pattern += "(?P<%s>\S+)\s+" % (field,) + re_pattern += r"(?P<%s>\S+)\s+" % (field,) matchobj = re.search(re_pattern, out) self.assertIsNotNone( diff --git a/psutil/tests/test_contracts.py b/psutil/tests/test_contracts.py index 877a5c066..d936eaf88 100755 --- a/psutil/tests/test_contracts.py +++ b/psutil/tests/test_contracts.py @@ -24,6 +24,7 @@ from psutil import MACOS from psutil import NETBSD from psutil import OPENBSD +from psutil import OSX from psutil import POSIX from psutil import SUNOS from psutil import WINDOWS @@ -512,7 +513,7 @@ def memory_full_info(self, ret, proc): value = getattr(ret, name) self.assertIsInstance(value, (int, long)) self.assertGreaterEqual(value, 0, msg=(name, value)) - if LINUX and name in ('vms', 'data'): + if LINUX or OSX and name in ('vms', 'data'): # On Linux there are processes (e.g. 'goa-daemon') whose # VMS is incredibly high for some reason. continue diff --git a/psutil/tests/test_posix.py b/psutil/tests/test_posix.py index 35f73eb80..5a8fdc173 100755 --- a/psutil/tests/test_posix.py +++ b/psutil/tests/test_posix.py @@ -185,6 +185,9 @@ def test_name(self): # be "pythonX.Y". name_ps = re.sub(r"\d.\d", "", name_ps) name_psutil = re.sub(r"\d.\d", "", name_psutil) + # ...may also be "python.X" + name_ps = re.sub(r"\d", "", name_ps) + name_psutil = re.sub(r"\d", "", name_psutil) self.assertEqual(name_ps, name_psutil) def test_name_long(self): diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index a4adf367f..aba8cdb45 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -28,6 +28,7 @@ from psutil import MACOS from psutil import NETBSD from psutil import OPENBSD +from psutil import OSX from psutil import POSIX from psutil import SUNOS from psutil import WINDOWS @@ -600,6 +601,8 @@ def test_memory_full_info(self): for name in mem._fields: value = getattr(mem, name) self.assertGreaterEqual(value, 0, msg=(name, value)) + if name == 'vms' and OSX or LINUX: + continue self.assertLessEqual(value, total, msg=(name, value, total)) if LINUX or WINDOWS or MACOS: self.assertGreaterEqual(mem.uss, 0) diff --git a/psutil/tests/test_unicode.py b/psutil/tests/test_unicode.py index 4144b5c25..71b068c74 100755 --- a/psutil/tests/test_unicode.py +++ b/psutil/tests/test_unicode.py @@ -75,6 +75,7 @@ from psutil.tests import HAS_ENVIRON from psutil.tests import HAS_MEMORY_MAPS from psutil.tests import mock +from psutil.tests import PYPY from psutil.tests import reap_children from psutil.tests import run_test_module_by_name from psutil.tests import safe_mkdir @@ -285,6 +286,8 @@ def normpath(p): self.assertIsInstance(path, str) +# https://travis-ci.org/giampaolo/psutil/jobs/440073249 +@unittest.skipIf(PYPY and TRAVIS, "unreliable on PYPY + TRAVIS") @unittest.skipIf(MACOS and TRAVIS, "unreliable on TRAVIS") # TODO @unittest.skipIf(ASCII_FS, "ASCII fs") @unittest.skipIf(not subprocess_supports_unicode(TESTFN_UNICODE), From 2e220c8ae1b69299eb07bcbd419cf2f59837853d Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 12 Oct 2018 17:56:58 +0200 Subject: [PATCH 140/182] fix #715: do not print exception on import time in case cpu_times() fails. --- HISTORY.rst | 1 + psutil/__init__.py | 3 --- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index a03b5ad25..735b9345f 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -15,6 +15,7 @@ XXXX-XX-XX **Bug fixes** +- 715_: do not print exception on import time in case cpu_times() fails. - 1294_: [Windows] psutil.Process().connections() may sometimes fail with intermittent 0xC0000001. (patch by Sylvain Duchesne) - 1320_: [AIX] system CPU times (psutil.cpu_times()) were being reported with diff --git a/psutil/__init__.py b/psutil/__init__.py index 23dc1227e..f43ef7b6c 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -32,7 +32,6 @@ import subprocess import sys import time -import traceback try: import pwd except ImportError: @@ -1609,14 +1608,12 @@ def cpu_times(percpu=False): except Exception: # Don't want to crash at import time. _last_cpu_times = None - traceback.print_exc() try: _last_per_cpu_times = cpu_times(percpu=True) except Exception: # Don't want to crash at import time. _last_per_cpu_times = None - traceback.print_exc() def _cpu_tot_time(times): From 5404b4fabb34de20be65638c64c2c275f1580cd4 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 12 Oct 2018 19:55:47 +0200 Subject: [PATCH 141/182] fix travis --- psutil/tests/test_linux.py | 4 +--- psutil/tests/test_process.py | 5 +++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index c162666c2..4e4bd5650 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -1128,8 +1128,7 @@ def test_boot_time(self): psutil_value = psutil.boot_time() self.assertEqual(int(vmstat_value), int(psutil_value)) - @mock.patch('psutil.traceback.print_exc') - def test_no_procfs_on_import(self, tb): + def test_no_procfs_on_import(self): my_procfs = tempfile.mkdtemp() with open(os.path.join(my_procfs, 'stat'), 'w') as f: @@ -1148,7 +1147,6 @@ def open_mock(name, *args, **kwargs): patch_point = 'builtins.open' if PY3 else '__builtin__.open' with mock.patch(patch_point, side_effect=open_mock): reload_module(psutil) - assert tb.called self.assertRaises(IOError, psutil.cpu_times) self.assertRaises(IOError, psutil.cpu_times, percpu=True) diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index aba8cdb45..2126e32ad 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -899,8 +899,9 @@ def test_cpu_affinity(self): self.assertRaises(TypeError, p.cpu_affinity, 1) p.cpu_affinity(initial) # it should work with all iterables, not only lists - p.cpu_affinity(set(all_cpus)) - p.cpu_affinity(tuple(all_cpus)) + if not TRAVIS: + p.cpu_affinity(set(all_cpus)) + p.cpu_affinity(tuple(all_cpus)) @unittest.skipIf(not HAS_CPU_AFFINITY, 'not supported') def test_cpu_affinity_errs(self): From 8163eb1d797a2f5ab4575534a54b6dda0d027b38 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 12 Oct 2018 20:20:54 +0200 Subject: [PATCH 142/182] skip test on PYPY + Travis --- psutil/tests/test_unicode.py | 1 + 1 file changed, 1 insertion(+) diff --git a/psutil/tests/test_unicode.py b/psutil/tests/test_unicode.py index 71b068c74..cfa8dd92c 100755 --- a/psutil/tests/test_unicode.py +++ b/psutil/tests/test_unicode.py @@ -309,6 +309,7 @@ def expect_exact_path_match(cls): return cls.funky_name in os.listdir(here) +@unittest.skipIf(PYPY and TRAVIS, "unreliable on PYPY + TRAVIS") @unittest.skipIf(MACOS and TRAVIS, "unreliable on TRAVIS") # TODO @unittest.skipIf(not subprocess_supports_unicode(INVALID_NAME), "subprocess can't deal with invalid unicode") From 63aa2e7784de76ab39de8c5235a2333b4fb5c51b Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 13 Oct 2018 14:58:54 +0200 Subject: [PATCH 143/182] travis: disable pypy; se py 3.7 on osx --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2067f8d2f..84c340f4d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,10 +17,10 @@ matrix: env: PYVER=py27 - language: generic os: osx - env: PYVER=py36 + env: PYVER=py37 # pypy - - python: pypy - - python: pypy3 + # - python: pypy + # - python: pypy3 install: - ./.ci/travis/install.sh script: From 8c414c48d9db0f32194d8ac31ac7654e4483e024 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 16 Oct 2018 19:57:53 +0200 Subject: [PATCH 144/182] travis / osx: set py 3.6 --- .travis.yml | 2 +- psutil/_psutil_osx.c | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 84c340f4d..fdab64f62 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,7 +17,7 @@ matrix: env: PYVER=py27 - language: generic os: osx - env: PYVER=py37 + env: PYVER=py36 # pypy # - python: pypy # - python: pypy3 diff --git a/psutil/_psutil_osx.c b/psutil/_psutil_osx.c index b75c78b20..0db93fce3 100644 --- a/psutil/_psutil_osx.c +++ b/psutil/_psutil_osx.c @@ -713,10 +713,10 @@ psutil_virtual_mem(PyObject *self, PyObject *args) { return Py_BuildValue( "KKKKK", total, - (unsigned long long) vm.active_count * pagesize, - (unsigned long long) vm.inactive_count * pagesize, - (unsigned long long) vm.wire_count * pagesize, - // this is how vm_stat cmd does it + (unsigned long long) vm.active_count * pagesize, // active + (unsigned long long) vm.inactive_count * pagesize, // inactive + (unsigned long long) vm.wire_count * pagesize, // wired + // free mem; this is how vm_stat cmd does it (unsigned long long) (vm.free_count - vm.speculative_count) * pagesize ); } From 2f9dcf3c7b37d4da155df62bc835ddbdfe10b24f Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 16 Oct 2018 20:28:34 +0200 Subject: [PATCH 145/182] fix #1277 / osx / virtual_memory: 'available' and 'used' memory were not calculated properly --- HISTORY.rst | 2 ++ psutil/_psosx.py | 11 +++++++++-- psutil/_psutil_osx.c | 6 +++--- psutil/tests/test_osx.py | 6 ------ 4 files changed, 14 insertions(+), 11 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 735b9345f..4b2e7ed2f 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -16,6 +16,8 @@ XXXX-XX-XX **Bug fixes** - 715_: do not print exception on import time in case cpu_times() fails. +- 1277_: [OSX] available and used memory (psutil.virtual_memory()) metrics are + not accurate. - 1294_: [Windows] psutil.Process().connections() may sometimes fail with intermittent 0xC0000001. (patch by Sylvain Duchesne) - 1320_: [AIX] system CPU times (psutil.cpu_times()) were being reported with diff --git a/psutil/_psosx.py b/psutil/_psosx.py index fbfedf3ea..94e22bc72 100644 --- a/psutil/_psosx.py +++ b/psutil/_psosx.py @@ -119,9 +119,16 @@ def virtual_memory(): """System virtual memory as a namedtuple.""" - total, active, inactive, wired, free = cext.virtual_mem() + total, active, inactive, wired, free, speculative = cext.virtual_mem() + # This is how Zabbix calculate avail and used mem: + # https://github.com/zabbix/zabbix/blob/trunk/src/libs/zbxsysinfo/ + # osx/memory.c + # Also see: https://github.com/giampaolo/psutil/issues/1277 avail = inactive + free - used = active + inactive + wired + used = active + wired + # This is NOT how Zabbix calculates free mem but it matches "free" + # cmdline utility. + free -= speculative percent = usage_percent((total - avail), total, round_=1) return svmem(total, avail, percent, used, free, active, inactive, wired) diff --git a/psutil/_psutil_osx.c b/psutil/_psutil_osx.c index 0db93fce3..be08de552 100644 --- a/psutil/_psutil_osx.c +++ b/psutil/_psutil_osx.c @@ -711,13 +711,13 @@ psutil_virtual_mem(PyObject *self, PyObject *args) { return NULL; return Py_BuildValue( - "KKKKK", + "KKKKKK", total, (unsigned long long) vm.active_count * pagesize, // active (unsigned long long) vm.inactive_count * pagesize, // inactive (unsigned long long) vm.wire_count * pagesize, // wired - // free mem; this is how vm_stat cmd does it - (unsigned long long) (vm.free_count - vm.speculative_count) * pagesize + (unsigned long long) vm.free_count * pagesize, // free + (unsigned long long) vm.speculative_count * pagesize // speculative ); } diff --git a/psutil/tests/test_osx.py b/psutil/tests/test_osx.py index 557af9f95..7aabebe67 100755 --- a/psutil/tests/test_osx.py +++ b/psutil/tests/test_osx.py @@ -225,12 +225,6 @@ def test_vmem_free(self): psutil_val = psutil.virtual_memory().free self.assertAlmostEqual(psutil_val, vmstat_val, delta=MEMORY_TOLERANCE) - @retry_before_failing() - def test_vmem_available(self): - vmstat_val = vm_stat("inactive") + vm_stat("free") - psutil_val = psutil.virtual_memory().available - self.assertAlmostEqual(psutil_val, vmstat_val, delta=MEMORY_TOLERANCE) - @retry_before_failing() def test_vmem_active(self): vmstat_val = vm_stat("active") From 8b465260638cde5370808e23eb9ace608141393a Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 16 Oct 2018 21:17:52 +0200 Subject: [PATCH 146/182] #1197 / linux / cpu_freq(): parse /proc/cpuinfo in case /sys/devices/system/cpu fs is not available --- HISTORY.rst | 2 ++ psutil/__init__.py | 16 ++++++++++++++-- psutil/_pslinux.py | 13 +++++++++++++ psutil/tests/test_linux.py | 29 +++++++++++++++++++++++++++++ 4 files changed, 58 insertions(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 4b2e7ed2f..448370948 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -7,6 +7,8 @@ XXXX-XX-XX **Enhancements** +- 1197_: [Linux] cpu_freq() is now implemented by parsing /proc/cpuinfo in case + /sys/devices/system/cpu/* filesystem is not available. - 1310_: [Linux] psutil.sensors_temperatures() now parses /sys/class/thermal in case /sys/class/hwmon fs is not available (e.g. Raspberry Pi). (patch by Alex Manuskin) diff --git a/psutil/__init__.py b/psutil/__init__.py index f43ef7b6c..c2a83fb15 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -1861,13 +1861,25 @@ def cpu_freq(percpu=False): return ret[0] else: currs, mins, maxs = 0.0, 0.0, 0.0 + set_none = False for cpu in ret: currs += cpu.current + # On Linux if /proc/cpuinfo is used min/max are set + # to None. + if LINUX and cpu.min is None: + set_none = True + continue mins += cpu.min maxs += cpu.max + current = currs / num_cpus - min_ = mins / num_cpus - max_ = maxs / num_cpus + + if set_none: + min_ = max_ = None + else: + min_ = mins / num_cpus + max_ = maxs / num_cpus + return _common.scpufreq(current, min_, max_) __all__.append("cpu_freq") diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index a69b1d664..1520261c1 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -704,6 +704,19 @@ def cpu_freq(): ret.append(_common.scpufreq(curr, min_, max_)) return ret +elif os.path.exists("/proc/cpuinfo"): + def cpu_freq(): + """Alternate implementation using /proc/cpuinfo. + min and max frequencies are not available and are set to None. + """ + ret = [] + with open_binary('%s/cpuinfo' % get_procfs_path()) as f: + for line in f: + if line.lower().startswith(b'cpu mhz'): + key, value = line.split(b'\t:', 1) + ret.append(_common.scpufreq(float(value), None, None)) + return ret + # ===================================================================== # --- network diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index 4e4bd5650..115a6af8a 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -703,6 +703,35 @@ def glob_mock(pattern): assert psutil.cpu_freq() self.assertEqual(len(flags), 2) + @unittest.skipIf(not HAS_CPU_FREQ, "not supported") + def test_cpu_freq_use_cpuinfo(self): + # Emulate a case where /sys/devices/system/cpu/cpufreq* does not + # exist and /proc/cpuinfo is used instead. + def path_exists_mock(path): + if path.startswith('/sys/devices/system/cpu/'): + return False + else: + if path == "/proc/cpuinfo": + flags.append(None) + return os_path_exists(path) + + flags = [] + os_path_exists = os.path.exists + try: + with mock.patch("os.path.exists", side_effect=path_exists_mock): + reload_module(psutil._pslinux) + ret = psutil.cpu_freq() + assert ret + assert flags + self.assertIsNone(ret.min) + self.assertIsNone(ret.max) + for freq in psutil.cpu_freq(percpu=True): + self.assertIsNone(freq.min) + self.assertIsNone(freq.max) + finally: + reload_module(psutil._pslinux) + reload_module(psutil) + @unittest.skipIf(not HAS_CPU_FREQ, "not supported") def test_cpu_freq_emulate_data(self): def open_mock(name, *args, **kwargs): From 6db8d2e41dc67621eae9d8eeab3c39fef7e2ddf4 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 16 Oct 2018 22:18:33 +0200 Subject: [PATCH 147/182] refactor hasattr() checks as global constants --- psutil/_psbsd.py | 15 ++++++++++----- psutil/_pslinux.py | 4 +++- psutil/_pswindows.py | 3 ++- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/psutil/_psbsd.py b/psutil/_psbsd.py index 7f4bcb6de..c2896cb7c 100644 --- a/psutil/_psbsd.py +++ b/psutil/_psbsd.py @@ -103,6 +103,11 @@ PAGESIZE = os.sysconf("SC_PAGE_SIZE") AF_LINK = cext_posix.AF_LINK +HAS_PER_CPU_TIMES = hasattr(cext, "per_cpu_times") +HAS_PROC_NUM_THREADS = hasattr(cext, "proc_num_threads") +HAS_PROC_OPEN_FILES = hasattr(cext, 'proc_open_files') +HAS_PROC_NUM_FDS = hasattr(cext, 'proc_num_fds') + kinfo_proc_map = dict( ppid=0, status=1, @@ -211,7 +216,7 @@ def cpu_times(): return scputimes(user, nice, system, idle, irq) -if hasattr(cext, "per_cpu_times"): +if HAS_PER_CPU_TIMES: def per_cpu_times(): """Return system CPU times as a namedtuple""" ret = [] @@ -678,7 +683,7 @@ def create_time(self): @wrap_exceptions def num_threads(self): - if hasattr(cext, "proc_num_threads"): + if HAS_PROC_NUM_THREADS: # FreeBSD return cext.proc_num_threads(self.pid) else: @@ -798,7 +803,7 @@ def cwd(self): elif NETBSD: with wrap_exceptions_procfs(self): return os.readlink("/proc/%s/cwd" % self.pid) - elif hasattr(cext, 'proc_open_files'): + elif HAS_PROC_OPEN_FILES: # FreeBSD < 8 does not support functions based on # kinfo_getfile() and kinfo_getvmmap() return cext.proc_cwd(self.pid) or None @@ -817,7 +822,7 @@ def _not_implemented(self): # FreeBSD < 8 does not support functions based on kinfo_getfile() # and kinfo_getvmmap() - if hasattr(cext, 'proc_open_files'): + if HAS_PROC_OPEN_FILES: @wrap_exceptions def open_files(self): """Return files opened by process as a list of namedtuples.""" @@ -828,7 +833,7 @@ def open_files(self): # FreeBSD < 8 does not support functions based on kinfo_getfile() # and kinfo_getvmmap() - if hasattr(cext, 'proc_num_fds'): + if HAS_PROC_NUM_FDS: @wrap_exceptions def num_fds(self): """Return the number of file descriptors opened by this process.""" diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index 1520261c1..236934fc9 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -73,6 +73,7 @@ POWER_SUPPLY_PATH = "/sys/class/power_supply" HAS_SMAPS = os.path.exists('/proc/%s/smaps' % os.getpid()) HAS_PRLIMIT = hasattr(cext, "linux_prlimit") +HAS_PROC_IO_PRIORITY = hasattr(cext, "proc_ioprio_get") _DEFAULT = object() # RLIMIT_* constants, not guaranteed to be present on all kernels @@ -1928,7 +1929,7 @@ def cpu_affinity_set(self, cpus): raise # only starting from kernel 2.6.13 - if hasattr(cext, "proc_ioprio_get"): + if HAS_PROC_IO_PRIORITY: @wrap_exceptions def ionice_get(self): @@ -1971,6 +1972,7 @@ def ionice_set(self, ioclass, value): return cext.proc_ioprio_set(self.pid, ioclass, value) if HAS_PRLIMIT: + @wrap_exceptions def rlimit(self, resource, limits=None): # If pid is 0 prlimit() applies to the calling process and diff --git a/psutil/_pswindows.py b/psutil/_pswindows.py index 18651d6cf..b938d42ff 100644 --- a/psutil/_pswindows.py +++ b/psutil/_pswindows.py @@ -83,6 +83,7 @@ cext.ERROR_ACCESS_DENIED]) NO_SUCH_SERVICE_ERRSET = frozenset([cext.ERROR_INVALID_NAME, cext.ERROR_SERVICE_DOES_NOT_EXIST]) +HAS_PROC_IO_PRIORITY = hasattr(cext, "proc_io_priority_get") if enum is None: @@ -928,7 +929,7 @@ def nice_set(self, value): return cext.proc_priority_set(self.pid, value) # available on Windows >= Vista - if hasattr(cext, "proc_io_priority_get"): + if HAS_PROC_IO_PRIORITY: @wrap_exceptions def ionice_get(self): return cext.proc_io_priority_get(self.pid) From fbece8e45991e8cc070bfd5e9efcd4974227abcd Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 19 Oct 2018 14:27:56 +0200 Subject: [PATCH 148/182] fix #1307: [Linux] disk_partitions() does not honour PROCFS_PATH --- HISTORY.rst | 1 + psutil/_pslinux.py | 11 +++++++++-- psutil/_psutil_linux.c | 9 ++++++--- psutil/tests/test_connections.py | 1 + psutil/tests/test_linux.py | 12 ++++++++++++ psutil/tests/test_system.py | 6 +----- 6 files changed, 30 insertions(+), 10 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 448370948..a02b8e2c0 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -22,6 +22,7 @@ XXXX-XX-XX not accurate. - 1294_: [Windows] psutil.Process().connections() may sometimes fail with intermittent 0xC0000001. (patch by Sylvain Duchesne) +- 1307_: [Linux] disk_partitions() does not honour PROCFS_PATH. - 1320_: [AIX] system CPU times (psutil.cpu_times()) were being reported with ticks unit as opposed to seconds. (patch by Jaime Fullaondo) - 1332_: [OSX] psutil debug messages are erroneously printed all the time. diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index 236934fc9..82e321899 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -1140,7 +1140,8 @@ def read_sysfs(): def disk_partitions(all=False): """Return mounted disk partitions as a list of namedtuples.""" fstypes = set() - with open_text("%s/filesystems" % get_procfs_path()) as f: + procfs_path = get_procfs_path() + with open_text("%s/filesystems" % procfs_path) as f: for line in f: line = line.strip() if not line.startswith("nodev"): @@ -1151,8 +1152,14 @@ def disk_partitions(all=False): if fstype == "zfs": fstypes.add("zfs") + # See: https://github.com/giampaolo/psutil/issues/1307 + if procfs_path == "/proc": + mtab_path = os.path.realpath("/etc/mtab") + else: + mtab_path = os.path.realpath("%s/self/mounts" % procfs_path) + retlist = [] - partitions = cext.disk_partitions() + partitions = cext.disk_partitions(mtab_path) for partition in partitions: device, mountpoint, fstype, opts = partition if device == 'none': diff --git a/psutil/_psutil_linux.c b/psutil/_psutil_linux.c index d1f0d1455..bd27b5f9c 100644 --- a/psutil/_psutil_linux.c +++ b/psutil/_psutil_linux.c @@ -195,6 +195,7 @@ static PyObject * psutil_disk_partitions(PyObject *self, PyObject *args) { FILE *file = NULL; struct mntent *entry; + const char *mtab_path; PyObject *py_dev = NULL; PyObject *py_mountp = NULL; PyObject *py_tuple = NULL; @@ -203,12 +204,14 @@ psutil_disk_partitions(PyObject *self, PyObject *args) { if (py_retlist == NULL) return NULL; - // MOUNTED constant comes from mntent.h and it's == '/etc/mtab' + if (!PyArg_ParseTuple(args, "s", &mtab_path)) + return NULL; + Py_BEGIN_ALLOW_THREADS - file = setmntent(MOUNTED, "r"); + file = setmntent(mtab_path, "r"); Py_END_ALLOW_THREADS if ((file == 0) || (file == NULL)) { - PyErr_SetFromErrnoWithFilename(PyExc_OSError, MOUNTED); + PyErr_SetFromErrnoWithFilename(PyExc_OSError, mtab_path); goto error; } diff --git a/psutil/tests/test_connections.py b/psutil/tests/test_connections.py index cba835e14..7f59a74cb 100755 --- a/psutil/tests/test_connections.py +++ b/psutil/tests/test_connections.py @@ -53,6 +53,7 @@ class Base(object): def setUp(self): + safe_rmpath(TESTFN) if not NETBSD: # NetBSD opens a UNIX socket to /var/log/run. cons = thisproc.connections(kind='all') diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index 115a6af8a..4b72f7257 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -1011,6 +1011,18 @@ def test_disk_partitions_mocked(self): assert ret self.assertEqual(ret[0].fstype, 'zfs') + def test_disk_partitions_procfs(self): + # See: https://github.com/giampaolo/psutil/issues/1307 + try: + with mock.patch('os.path.realpath', + return_value='/non/existent') as m: + with self.assertRaises(OSError) as cm: + psutil.disk_partitions() + assert m.called + self.assertEqual(cm.exception.errno, errno.ENOENT) + finally: + psutil.PROCFS_PATH = "/proc" + def test_disk_io_counters_kernel_2_4_mocked(self): # Tests /proc/diskstats parsing format for 2.4 kernels, see: # https://github.com/giampaolo/psutil/issues/767 diff --git a/psutil/tests/test_system.py b/psutil/tests/test_system.py index 8b07caff6..7cc678f05 100755 --- a/psutil/tests/test_system.py +++ b/psutil/tests/test_system.py @@ -522,11 +522,7 @@ def test_disk_partitions(self): if err.errno not in (errno.EPERM, errno.EACCES): raise else: - if SUNOS or TRAVIS: - # on solaris apparently mount points can also be files - assert os.path.exists(disk.mountpoint), disk - else: - assert os.path.isdir(disk.mountpoint), disk + assert os.path.exists(disk.mountpoint), disk self.assertIsInstance(disk.fstype, str) self.assertIsInstance(disk.opts, str) From 7300bd48b4ea6fd98309cc9da569d6015b1ac24a Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 19 Oct 2018 21:49:40 +0200 Subject: [PATCH 149/182] fix #1004: Process.io_counters() may raise ValueError --- HISTORY.rst | 1 + IDEAS | 5 +++++ psutil/_pslinux.py | 29 +++++++++++++++++++---------- 3 files changed, 25 insertions(+), 10 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index a02b8e2c0..71cff74ee 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -18,6 +18,7 @@ XXXX-XX-XX **Bug fixes** - 715_: do not print exception on import time in case cpu_times() fails. +- 1004_: [Linux] Process.io_counters() may raise ValueError. - 1277_: [OSX] available and used memory (psutil.virtual_memory()) metrics are not accurate. - 1294_: [Windows] psutil.Process().connections() may sometimes fail with diff --git a/IDEAS b/IDEAS index 8190d48b9..2178333fe 100644 --- a/IDEAS +++ b/IDEAS @@ -12,12 +12,17 @@ PLATFORMS - #82: Cygwin (PR at #998) - #276: GNU/Hurd - #693: Windows Nano +- #1251: Windows bash - DragonFlyBSD - HP-UX FEATURES ======== +- #1115: users() idle time. + +- #1102: Process.is64bit(). + - #371: sensors_temperatures() at least for macOS. - #669: Windows / net_if_addrs(): return broadcast addr. diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index 82e321899..71f0c9843 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -1650,18 +1650,27 @@ def io_counters(self): # https://github.com/giampaolo/psutil/issues/1004 line = line.strip() if line: - name, value = line.split(b': ') - fields[name] = int(value) + try: + name, value = line.split(b': ') + except ValueError: + # https://github.com/giampaolo/psutil/issues/1004 + continue + else: + fields[name] = int(value) if not fields: raise RuntimeError("%s file was empty" % fname) - return pio( - fields[b'syscr'], # read syscalls - fields[b'syscw'], # write syscalls - fields[b'read_bytes'], # read bytes - fields[b'write_bytes'], # write bytes - fields[b'rchar'], # read chars - fields[b'wchar'], # write chars - ) + try: + return pio( + fields[b'syscr'], # read syscalls + fields[b'syscw'], # write syscalls + fields[b'read_bytes'], # read bytes + fields[b'write_bytes'], # write bytes + fields[b'rchar'], # read chars + fields[b'wchar'], # write chars + ) + except KeyError as err: + raise ValueError("%r field was not found in %s; found fields " + "are %r" % (err[0], fname, fields)) else: def io_counters(self): raise NotImplementedError("couldn't find /proc/%s/io (kernel " From d8b05151e65f9348aff9b58da977abd8cacb2127 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 20 Oct 2018 20:52:40 +0200 Subject: [PATCH 150/182] sensors_temperatures() / linux: convert defaultdict to dict --- IDEAS | 1 + psutil/_pslinux.py | 2 +- setup.py | 3 ++- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/IDEAS b/IDEAS index 2178333fe..babc0e8be 100644 --- a/IDEAS +++ b/IDEAS @@ -169,3 +169,4 @@ RESOURCES - libstatgrab: http://www.i-scream.org/libstatgrab/ - top: http://www.unixtop.org/ - oshi: https://github.com/oshi/oshi +- netdata: https://github.com/netdata/netdata diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index 71f0c9843..ecc4c703f 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -1278,7 +1278,7 @@ def sensors_temperatures(): ret[unit_name].append(('', current, high, critical)) - return ret + return dict(ret) def sensors_fans(): diff --git a/setup.py b/setup.py index c4845a8d7..28cde6f12 100755 --- a/setup.py +++ b/setup.py @@ -284,7 +284,8 @@ def main(): 'ps', 'top', 'kill', 'free', 'lsof', 'netstat', 'nice', 'tty', 'ionice', 'uptime', 'taskmgr', 'process', 'df', 'iotop', 'iostat', 'ifconfig', 'taskset', 'who', 'pidof', 'pmap', 'smem', 'pstree', - 'monitoring', 'ulimit', 'prlimit', 'smem', + 'monitoring', 'ulimit', 'prlimit', 'smem', 'performance', + 'metrics', 'agent', 'observability', ], author='Giampaolo Rodola', author_email='g.rodola@gmail.com', From 648d8ba39eff4867d461a45a77d1245e2a909234 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 30 Oct 2018 10:57:16 +0100 Subject: [PATCH 151/182] pre release --- HISTORY.rst | 2 +- docs/index.rst | 6 +++++- psutil/_psutil_windows.c | 3 ++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 71cff74ee..5a270184f 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -3,7 +3,7 @@ 5.4.8 ===== -XXXX-XX-XX +2018-10-30 **Enhancements** diff --git a/docs/index.rst b/docs/index.rst index 7763b6de0..91abffe84 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -702,7 +702,7 @@ Sensors See also `temperatures.py `__ and `sensors.py `__ for an example application. - Availability: Linux, macOS + Availability: Linux .. versionadded:: 5.1.0 @@ -2638,6 +2638,10 @@ take a look at the Timeline ======== +- 2018-10-30: + `5.4.8 `__ - + `what's new `__ - + `diff `__ - 2018-08-14: `5.4.7 `__ - `what's new `__ - diff --git a/psutil/_psutil_windows.c b/psutil/_psutil_windows.c index 4acea3600..edb5996c4 100644 --- a/psutil/_psutil_windows.c +++ b/psutil/_psutil_windows.c @@ -2431,7 +2431,8 @@ psutil_net_io_counters(PyObject *self, PyObject *args) { /* - * Return a Python dict of tuples for disk I/O information + * Return a Python dict of tuples for disk I/O information. This may + * require running "diskperf -y" command first. */ static PyObject * psutil_disk_io_counters(PyObject *self, PyObject *args) { From bb5d032be76980a9e110f03f1203bd35fa85a793 Mon Sep 17 00:00:00 2001 From: Alex Manuskin Date: Sat, 3 Nov 2018 16:06:13 +0200 Subject: [PATCH 152/182] FreeBSD adding temperature sensors (WIP) (#1350) FreeBSD: add temperature sensors --- psutil/_psbsd.py | 18 +++++++++++++++++ psutil/_psutil_bsd.c | 2 ++ psutil/arch/freebsd/specific.c | 36 ++++++++++++++++++++++++++++++++++ psutil/arch/freebsd/specific.h | 1 + psutil/tests/test_bsd.py | 17 ++++++++++++++++ psutil/tests/test_contracts.py | 4 ++-- 6 files changed, 76 insertions(+), 2 deletions(-) diff --git a/psutil/_psbsd.py b/psutil/_psbsd.py index c2896cb7c..0727dd2e8 100644 --- a/psutil/_psbsd.py +++ b/psutil/_psbsd.py @@ -11,6 +11,7 @@ import xml.etree.ElementTree as ET from collections import namedtuple from socket import AF_INET +from collections import defaultdict from . import _common from . import _psposix @@ -437,6 +438,23 @@ def sensors_battery(): secsleft = minsleft * 60 return _common.sbattery(percent, secsleft, power_plugged) + def sensors_temperatures(): + "Return CPU cores temperatures if available, else an empty dict." + ret = defaultdict(list) + num_cpus = cpu_count_logical() + for cpu in range(num_cpus): + try: + current, high = cext.sensors_cpu_temperature(cpu) + if high <= 0: + high = None + name = "Core %s" % cpu + ret["coretemp"].append( + _common.shwtemp(name, current, high, high)) + except NotImplementedError: + pass + + return ret + # ===================================================================== # --- other system functions diff --git a/psutil/_psutil_bsd.c b/psutil/_psutil_bsd.c index 4e91c02ed..6b366f13b 100644 --- a/psutil/_psutil_bsd.c +++ b/psutil/_psutil_bsd.c @@ -981,6 +981,8 @@ PsutilMethods[] = { #if defined(PSUTIL_FREEBSD) {"sensors_battery", psutil_sensors_battery, METH_VARARGS, "Return battery information."}, + {"sensors_cpu_temperature", psutil_sensors_cpu_temperature, METH_VARARGS, + "Return temperature information for a given CPU core number."}, #endif // --- others diff --git a/psutil/arch/freebsd/specific.c b/psutil/arch/freebsd/specific.c index 177ac8a6b..14be26b36 100644 --- a/psutil/arch/freebsd/specific.c +++ b/psutil/arch/freebsd/specific.c @@ -31,6 +31,7 @@ #define PSUTIL_TV2DOUBLE(t) ((t).tv_sec + (t).tv_usec / 1000000.0) #define PSUTIL_BT2MSEC(bt) (bt.sec * 1000 + (((uint64_t) 1000000000 * (uint32_t) \ (bt.frac >> 32) ) >> 32 ) / 1000000) +#define DECIKELVIN_2_CELCIUS(t) (t - 2731) / 10 #ifndef _PATH_DEVNULL #define _PATH_DEVNULL "/dev/null" #endif @@ -1010,3 +1011,38 @@ psutil_sensors_battery(PyObject *self, PyObject *args) { PyErr_SetFromErrno(PyExc_OSError); return NULL; } + + +/* + * Return temperature information for a given CPU core number. + */ +PyObject * +psutil_sensors_cpu_temperature(PyObject *self, PyObject *args) { + int current; + int tjmax; + int core; + char sensor[26]; + size_t size = sizeof(current); + + if (! PyArg_ParseTuple(args, "i", &core)) + return NULL; + sprintf(sensor, "dev.cpu.%d.temperature", core); + if (sysctlbyname(sensor, ¤t, &size, NULL, 0)) + goto error; + current = DECIKELVIN_2_CELCIUS(current); + + // Return -273 in case of faliure. + sprintf(sensor, "dev.cpu.%d.coretemp.tjmax", core); + if (sysctlbyname(sensor, &tjmax, &size, NULL, 0)) + tjmax = 0; + tjmax = DECIKELVIN_2_CELCIUS(tjmax); + + return Py_BuildValue("ii", current, tjmax); + +error: + if (errno == ENOENT) + PyErr_SetString(PyExc_NotImplementedError, "no temperature sensors"); + else + PyErr_SetFromErrno(PyExc_OSError); + return NULL; +} diff --git a/psutil/arch/freebsd/specific.h b/psutil/arch/freebsd/specific.h index 0df66eccb..cb71ff612 100644 --- a/psutil/arch/freebsd/specific.h +++ b/psutil/arch/freebsd/specific.h @@ -29,4 +29,5 @@ PyObject* psutil_virtual_mem(PyObject* self, PyObject* args); PyObject* psutil_cpu_stats(PyObject* self, PyObject* args); #if defined(PSUTIL_FREEBSD) PyObject* psutil_sensors_battery(PyObject* self, PyObject* args); +PyObject* psutil_sensors_cpu_temperature(PyObject* self, PyObject* args); #endif diff --git a/psutil/tests/test_bsd.py b/psutil/tests/test_bsd.py index 7846c1ca2..df43a023e 100755 --- a/psutil/tests/test_bsd.py +++ b/psutil/tests/test_bsd.py @@ -427,6 +427,23 @@ def test_sensors_battery_no_battery(self): sysctl("hw.acpi.acline") self.assertIsNone(psutil.sensors_battery()) + # --- sensors_temperatures + + def test_sensors_temperatures_against_sysctl(self): + num_cpus = psutil.cpu_count(True) + for cpu in range(num_cpus): + sensor = "dev.cpu.%s.temperature" % cpu + # sysctl returns a string in the format 46.0C + sysctl_result = int(float(sysctl(sensor)[:-1])) + self.assertAlmostEqual( + psutil.sensors_temperatures()["coretemp"][cpu].current, + sysctl_result, delta=10) + + sensor = "dev.cpu.%s.coretemp.tjmax" % cpu + sysctl_result = int(float(sysctl(sensor)[:-1])) + self.assertEqual( + psutil.sensors_temperatures()["coretemp"][cpu].high, + sysctl_result) # ===================================================================== # --- OpenBSD diff --git a/psutil/tests/test_contracts.py b/psutil/tests/test_contracts.py index d936eaf88..8ff41e5b3 100755 --- a/psutil/tests/test_contracts.py +++ b/psutil/tests/test_contracts.py @@ -119,7 +119,7 @@ def test_cpu_freq(self): def test_sensors_temperatures(self): self.assertEqual( - hasattr(psutil, "sensors_temperatures"), LINUX) + hasattr(psutil, "sensors_temperatures"), LINUX or FREEBSD) def test_sensors_fans(self): self.assertEqual(hasattr(psutil, "sensors_fans"), LINUX) @@ -337,7 +337,7 @@ def test_fetch_all(self): self.assertEqual(err.name, p.name()) assert str(err) assert err.msg - except Exception as err: + except Exception: s = '\n' + '=' * 70 + '\n' s += "FAIL: test_%s (proc=%s" % (name, p) if ret != default: From 7ddfa9a637f56dbe236dde285991569783fe444e Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 3 Nov 2018 15:45:50 +0100 Subject: [PATCH 153/182] #1350: give credits to @amanusk --- CREDITS | 2 +- HISTORY.rst | 7 +++++++ docs/index.rst | 17 ++--------------- psutil/__init__.py | 2 +- 4 files changed, 11 insertions(+), 17 deletions(-) diff --git a/CREDITS b/CREDITS index b3c8164c9..404310f50 100644 --- a/CREDITS +++ b/CREDITS @@ -546,7 +546,7 @@ I: 1278 N: Alex Manuskin W: https://github.com/amanusk -I: 1284, 1345 +I: 1284, 1345, 1350 N: Sylvain Duchesne W: https://github.com/sylvainduchesne diff --git a/HISTORY.rst b/HISTORY.rst index 5a270184f..97f7b5227 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,5 +1,12 @@ *Bug tracker at https://github.com/giampaolo/psutil/issues* +5.5.0 +===== + +**Enhancements** + +- 1350_: [FreeBSD] added support for sensors_temperatures(). + 5.4.8 ===== diff --git a/docs/index.rst b/docs/index.rst index 91abffe84..d54129c24 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -702,14 +702,11 @@ Sensors See also `temperatures.py `__ and `sensors.py `__ for an example application. - Availability: Linux + Availability: Linux, FreeBSD .. versionadded:: 5.1.0 - .. warning:: - - this API is experimental. Backward incompatible changes may occur if - deemed necessary. + .. versionchanged:: 5.5.0 added FreeBSD support .. function:: sensors_fans() @@ -730,11 +727,6 @@ Sensors .. versionadded:: 5.2.0 - .. warning:: - - this API is experimental. Backward incompatible changes may occur if - deemed necessary. - .. function:: sensors_battery() Return battery status information as a named tuple including the following @@ -774,11 +766,6 @@ Sensors .. versionchanged:: 5.4.2 added macOS support - .. warning:: - - this API is experimental. Backward incompatible changes may occur if - deemed necessary. - Other system info ----------------- diff --git a/psutil/__init__.py b/psutil/__init__.py index c2a83fb15..5a5720d7a 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -220,7 +220,7 @@ ] __all__.extend(_psplatform.__extra__all__) __author__ = "Giampaolo Rodola'" -__version__ = "5.4.8" +__version__ = "5.5.0" version_info = tuple([int(num) for num in __version__.split('.')]) AF_LINK = _psplatform.AF_LINK POWER_TIME_UNLIMITED = _common.POWER_TIME_UNLIMITED From 8f99f3782663959062ee868bbfdbc336307a3a4d Mon Sep 17 00:00:00 2001 From: Koen Kooi Date: Mon, 5 Nov 2018 15:17:16 +0000 Subject: [PATCH 154/182] Fix #1354 [Linux] disk_io_counters() fails on Linux kernel 4.18+ (#1360) Linux kernel 4.18+ added 4 fields, ingore them and parse the rest as usual. --- psutil/_pslinux.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index ecc4c703f..b775d39ae 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -1060,6 +1060,8 @@ def read_procfs(): # ...unless (Linux 2.6) the line refers to a partition instead # of a disk, in which case the line has less fields (7): # "3 1 hda1 8 8 8 8" + # 4.18+ has 4 fields added: + # "3 0 hda 8 8 8 8 8 8 8 8 8 8 8 0 0 0 0" # See: # https://www.kernel.org/doc/Documentation/iostats.txt # https://www.kernel.org/doc/Documentation/ABI/testing/procfs-diskstats @@ -1074,7 +1076,7 @@ def read_procfs(): reads = int(fields[2]) (reads_merged, rbytes, rtime, writes, writes_merged, wbytes, wtime, _, busy_time, _) = map(int, fields[4:14]) - elif flen == 14: + elif flen == 14 or flen == 18: # Linux 2.6+, line referring to a disk name = fields[2] (reads, reads_merged, rbytes, rtime, writes, writes_merged, From 76682c0cfdf1bfe0db8993f261c7ab761643af93 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 5 Nov 2018 16:20:54 +0100 Subject: [PATCH 155/182] give credits to @koenkooi for #1360 --- CREDITS | 4 ++++ HISTORY.rst | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/CREDITS b/CREDITS index 404310f50..ee06df0d7 100644 --- a/CREDITS +++ b/CREDITS @@ -563,3 +563,7 @@ I: 1332 N: Jaime Fullaondo W: https://github.com/truthbk I: 1320 + +N: Koen Kooi +W: https://github.com/koenkooi +I: 1360 diff --git a/HISTORY.rst b/HISTORY.rst index 97f7b5227..56d33296a 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -7,6 +7,10 @@ - 1350_: [FreeBSD] added support for sensors_temperatures(). +**Bug fixes** + +- 1354_: [Linux] disk_io_counters() fails on Linux kernel 4.18+. + 5.4.8 ===== From e56c7ed2990117edccf1803a56b509b2518f8230 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 7 Nov 2018 13:12:45 +0100 Subject: [PATCH 156/182] fix PEP8 style mistakes --- psutil/tests/__init__.py | 2 +- psutil/tests/test_aix.py | 14 +++++++------- psutil/tests/test_osx.py | 2 +- scripts/internal/check_broken_links.py | 2 +- scripts/internal/print_announce.py | 2 +- scripts/internal/winmake.py | 2 +- scripts/winservices.py | 2 +- 7 files changed, 13 insertions(+), 13 deletions(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 437588a6f..eb49b9c2b 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -602,7 +602,7 @@ def __init__(self, timeout=None, retries=None, interval=0.001, - logfun=lambda s: print(s, file=sys.stderr), + logfun=print, ): if timeout and retries: raise ValueError("timeout and retries args are mutually exclusive") diff --git a/psutil/tests/test_aix.py b/psutil/tests/test_aix.py index 0b29215f0..4f0da4e00 100755 --- a/psutil/tests/test_aix.py +++ b/psutil/tests/test_aix.py @@ -56,10 +56,10 @@ def test_swap_memory(self): # we'll always have 'MB' in the result # TODO maybe try to use "swap -l" to check "used" too, but its units # are not guaranteed to be "MB" so parsing may not be consistent - matchobj = re.search("(?P\S+)\s+" - "(?P\S+)\s+" - "(?P\S+)\s+" - "(?P\d+)MB", out) + matchobj = re.search(r"(?P\S+)\s+" + r"(?P\S+)\s+" + r"(?P\S+)\s+" + r"(?P\d+)MB", out) self.assertIsNotNone( matchobj, "lsps command returned unexpected output") @@ -74,11 +74,11 @@ def test_swap_memory(self): def test_cpu_stats(self): out = sh('/usr/bin/mpstat -a') - re_pattern = "ALL\s*" + re_pattern = r"ALL\s*" for field in ("min maj mpcs mpcr dev soft dec ph cs ics bound rq " "push S3pull S3grd S0rd S1rd S2rd S3rd S4rd S5rd " "sysc").split(): - re_pattern += "(?P<%s>\S+)\s+" % (field,) + re_pattern += r"(?P<%s>\S+)\s+" % (field,) matchobj = re.search(re_pattern, out) self.assertIsNotNone( @@ -106,7 +106,7 @@ def test_cpu_stats(self): def test_cpu_count_logical(self): out = sh('/usr/bin/mpstat -a') - mpstat_lcpu = int(re.search("lcpu=(\d+)", out).group(1)) + mpstat_lcpu = int(re.search(r"lcpu=(\d+)", out).group(1)) psutil_lcpu = psutil.cpu_count(logical=True) self.assertEqual(mpstat_lcpu, psutil_lcpu) diff --git a/psutil/tests/test_osx.py b/psutil/tests/test_osx.py index 7aabebe67..cf5c5ea8f 100755 --- a/psutil/tests/test_osx.py +++ b/psutil/tests/test_osx.py @@ -285,7 +285,7 @@ def test_net_if_stats(self): @unittest.skipIf(not HAS_BATTERY, "no battery") def test_sensors_battery(self): out = sh("pmset -g batt") - percent = re.search("(\d+)%", out).group(1) + percent = re.search(r"(\d+)%", out).group(1) drawing_from = re.search("Now drawing from '([^']+)'", out).group(1) power_plugged = drawing_from == "AC Power" psutil_result = psutil.sensors_battery() diff --git a/scripts/internal/check_broken_links.py b/scripts/internal/check_broken_links.py index 7cf1e4898..3d108d810 100755 --- a/scripts/internal/check_broken_links.py +++ b/scripts/internal/check_broken_links.py @@ -150,7 +150,7 @@ def parse_c(fname): subidx = i + 1 while True: nextline = lines[subidx].strip() - if re.match('^\* .+', nextline): + if re.match(r'^\* .+', nextline): url += nextline[1:].strip() else: break diff --git a/scripts/internal/print_announce.py b/scripts/internal/print_announce.py index c92cbcb2e..a39351d61 100755 --- a/scripts/internal/print_announce.py +++ b/scripts/internal/print_announce.py @@ -81,7 +81,7 @@ def get_changes(): for i, line in enumerate(lines): line = lines.pop(0) line = line.rstrip() - if re.match("^- \d+_: ", line): + if re.match(r"^- \d+_: ", line): num, _, rest = line.partition(': ') num = ''.join([x for x in num if x.isdigit()]) line = "- #%s: %s" % (num, rest) diff --git a/scripts/internal/winmake.py b/scripts/internal/winmake.py index c4722a029..b1ce7b8a4 100755 --- a/scripts/internal/winmake.py +++ b/scripts/internal/winmake.py @@ -495,7 +495,7 @@ def set_python(s): '26-64', '27-64', '34-64', '35-64', '36-64', '37-64') for v in vers: if s == v: - path = 'C:\\python%s\python.exe' % s + path = r'C:\\python%s\python.exe' % s if os.path.isfile(path): print(path) PYTHON = path diff --git a/scripts/winservices.py b/scripts/winservices.py index 1a65adcef..b52aaf21d 100755 --- a/scripts/winservices.py +++ b/scripts/winservices.py @@ -4,7 +4,7 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -""" +r""" List all Windows services installed. $ python scripts/winservices.py From e2596ab17b8ae8321d8d82fb501737747f6cd31a Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 7 Nov 2018 13:13:55 +0100 Subject: [PATCH 157/182] disable false positive mem test on travis + osx --- psutil/tests/test_memory_leaks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/psutil/tests/test_memory_leaks.py b/psutil/tests/test_memory_leaks.py index ce0824596..fc3a0365f 100755 --- a/psutil/tests/test_memory_leaks.py +++ b/psutil/tests/test_memory_leaks.py @@ -524,6 +524,7 @@ def test_pids(self): # --- net + @unittest.skipIf(TRAVIS and MACOS, "false positive on travis") @skip_if_linux() def test_net_io_counters(self): self.execute(psutil.net_io_counters, nowrap=False) From fbe821ebccb61def22cea953f1df83134fa01d0e Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 8 Nov 2018 18:35:59 +0100 Subject: [PATCH 158/182] #1359: add test case for cpu_count(logical=False) against lscpu utility --- psutil/tests/test_linux.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index 4b72f7257..0f981b6bc 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -639,6 +639,16 @@ def test_cpu_count_logical_w_lscpu(self): num = len([x for x in out.split('\n') if not x.startswith('#')]) self.assertEqual(psutil.cpu_count(logical=True), num) + @unittest.skipIf(not which("lscpu"), "lscpu utility not available") + def test_cpu_count_physical_w_lscpu(self): + out = sh("lscpu -p") + core_ids = set() + for line in out.split('\n'): + if not line.startswith('#'): + fields = line.split(',') + core_ids.add(fields[1]) + self.assertEqual(psutil.cpu_count(logical=False), len(core_ids)) + def test_cpu_count_logical_mocked(self): import psutil._pslinux original = psutil._pslinux.cpu_count_logical() From 459556dd1e2979cdee22177339ced0761caf4c83 Mon Sep 17 00:00:00 2001 From: Alex Manuskin Date: Sat, 1 Dec 2018 20:14:53 +0200 Subject: [PATCH 159/182] Add CPU frequency support for FreeBSD (#1369) Add CPU frequency support for FreeBSD (patch by @amanusk) --- psutil/_psbsd.py | 26 ++++++++++++++++++++++++++ psutil/_psutil_bsd.c | 2 ++ psutil/arch/freebsd/specific.c | 33 +++++++++++++++++++++++++++++++++ psutil/arch/freebsd/specific.h | 1 + psutil/tests/test_bsd.py | 18 ++++++++++++++++++ psutil/tests/test_contracts.py | 2 +- 6 files changed, 81 insertions(+), 1 deletion(-) diff --git a/psutil/_psbsd.py b/psutil/_psbsd.py index 0727dd2e8..d3ce7b5cc 100644 --- a/psutil/_psbsd.py +++ b/psutil/_psbsd.py @@ -455,6 +455,32 @@ def sensors_temperatures(): return ret + def cpu_freq(): + """ + Return frequency metrics for CPUs. Currently, only CPU 0 is supported + by FreeBSD, all other cores match the frequency of CPU 0. + """ + ret = [] + num_cpus = cpu_count_logical() + for cpu in range(num_cpus): + try: + current, available_freq = cext.cpu_frequency(cpu) + except NotImplementedError: + continue + min_freq = None + max_freq = None + if available_freq: + try: + min_freq = int(available_freq.split(" ")[-1].split("/")[0]) + except(IndexError, ValueError): + pass + try: + max_freq = int(available_freq.split(" ")[0].split("/")[0]) + except(IndexError, ValueError): + pass + ret.append(_common.scpufreq(current, min_freq, max_freq)) + return ret + # ===================================================================== # --- other system functions diff --git a/psutil/_psutil_bsd.c b/psutil/_psutil_bsd.c index 6b366f13b..dce157f55 100644 --- a/psutil/_psutil_bsd.c +++ b/psutil/_psutil_bsd.c @@ -983,6 +983,8 @@ PsutilMethods[] = { "Return battery information."}, {"sensors_cpu_temperature", psutil_sensors_cpu_temperature, METH_VARARGS, "Return temperature information for a given CPU core number."}, + {"cpu_frequency", psutil_cpu_freq, METH_VARARGS, + "Return frequency of a given CPU"}, #endif // --- others diff --git a/psutil/arch/freebsd/specific.c b/psutil/arch/freebsd/specific.c index 14be26b36..70dc7f2ce 100644 --- a/psutil/arch/freebsd/specific.c +++ b/psutil/arch/freebsd/specific.c @@ -1046,3 +1046,36 @@ psutil_sensors_cpu_temperature(PyObject *self, PyObject *args) { PyErr_SetFromErrno(PyExc_OSError); return NULL; } + + +/* + * Return frequency information of a given CPU + */ +PyObject * +psutil_cpu_freq(PyObject *self, PyObject *args) { + int current; + int core; + char sensor[26]; + char available_freq_levels[1000]; + size_t size = sizeof(current); + + if (! PyArg_ParseTuple(args, "i", &core)) + return NULL; + sprintf(sensor, "dev.cpu.%d.freq", core); + if (sysctlbyname(sensor, ¤t, &size, NULL, 0)) + goto error; + + size = sizeof(available_freq_levels); + // In case of failure, an empty string is returned + sprintf(sensor, "dev.cpu.%d.freq_levels", core); + sysctlbyname(sensor, &available_freq_levels, &size, NULL, 0); + + return Py_BuildValue("is", current, available_freq_levels); + +error: + if (errno == ENOENT) + PyErr_SetString(PyExc_NotImplementedError, "Unable to read frequency"); + else + PyErr_SetFromErrno(PyExc_OSError); + return NULL; +} diff --git a/psutil/arch/freebsd/specific.h b/psutil/arch/freebsd/specific.h index cb71ff612..875c81664 100644 --- a/psutil/arch/freebsd/specific.h +++ b/psutil/arch/freebsd/specific.h @@ -30,4 +30,5 @@ PyObject* psutil_cpu_stats(PyObject* self, PyObject* args); #if defined(PSUTIL_FREEBSD) PyObject* psutil_sensors_battery(PyObject* self, PyObject* args); PyObject* psutil_sensors_cpu_temperature(PyObject* self, PyObject* args); +PyObject* psutil_cpu_freq(PyObject* self, PyObject* args); #endif diff --git a/psutil/tests/test_bsd.py b/psutil/tests/test_bsd.py index df43a023e..87feb1473 100755 --- a/psutil/tests/test_bsd.py +++ b/psutil/tests/test_bsd.py @@ -250,6 +250,24 @@ def test_proc_cpu_times(self): if len(tested) != 2: raise RuntimeError("couldn't find lines match in procstat out") + # --- cpu_freq(); tests against sysctl + def test_cpu_frequency_against_sysctl(self): + # Currently only cpu 0 is frequency is supported in FreeBSD + # All other cores use the same frequency + sensor = "dev.cpu.0.freq" + sysctl_result = int(sysctl(sensor)) + self.assertEqual(psutil.cpu_freq().current, sysctl_result) + + sensor = "dev.cpu.0.freq_levels" + sysctl_result = sysctl(sensor) + # sysctl returns a string of the format: + # / /... + # Ordered highest available to lowest available + max_freq = int(sysctl_result.split()[0].split("/")[0]) + min_freq = int(sysctl_result.split()[-1].split("/")[0]) + self.assertEqual(psutil.cpu_freq().max, max_freq) + self.assertEqual(psutil.cpu_freq().min, min_freq) + # --- virtual_memory(); tests against sysctl @retry_before_failing() diff --git a/psutil/tests/test_contracts.py b/psutil/tests/test_contracts.py index 8ff41e5b3..78e6ba22f 100755 --- a/psutil/tests/test_contracts.py +++ b/psutil/tests/test_contracts.py @@ -115,7 +115,7 @@ def test_cpu_freq(self): (os.path.exists("/sys/devices/system/cpu/cpufreq") or os.path.exists("/sys/devices/system/cpu/cpu0/cpufreq"))) self.assertEqual(hasattr(psutil, "cpu_freq"), - linux or MACOS or WINDOWS) + linux or MACOS or WINDOWS or FREEBSD) def test_sensors_temperatures(self): self.assertEqual( From fd0b6d76a6c3319119f9ae932a965ac06c5855c4 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 1 Dec 2018 19:28:02 +0100 Subject: [PATCH 160/182] give CREDITS to @amanusk for #1369 / #1352 and update doc --- CREDITS | 2 +- HISTORY.rst | 7 +++++++ docs/index.rst | 3 ++- psutil/__init__.py | 2 +- psutil/_psbsd.py | 12 +++++------- psutil/arch/freebsd/specific.c | 10 +++++++--- psutil/tests/test_bsd.py | 5 ++--- 7 files changed, 25 insertions(+), 16 deletions(-) diff --git a/CREDITS b/CREDITS index ee06df0d7..30d9006f9 100644 --- a/CREDITS +++ b/CREDITS @@ -546,7 +546,7 @@ I: 1278 N: Alex Manuskin W: https://github.com/amanusk -I: 1284, 1345, 1350 +I: 1284, 1345, 1350, 1352 N: Sylvain Duchesne W: https://github.com/sylvainduchesne diff --git a/HISTORY.rst b/HISTORY.rst index 56d33296a..7cfe0f191 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,5 +1,12 @@ *Bug tracker at https://github.com/giampaolo/psutil/issues* +5.5.1 +===== + +**Enhancements** + +- 1352_: [FreeBSD] added support for CPU frequency. (patch by Alex Manuskin) + 5.5.0 ===== diff --git a/docs/index.rst b/docs/index.rst index d54129c24..85506a280 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -235,10 +235,11 @@ CPU scpufreq(current=1703.609, min=800.0, max=3500.0), scpufreq(current=1754.289, min=800.0, max=3500.0)] - Availability: Linux, macOS, Windows + Availability: Linux, macOS, Windows, FreeBSD .. versionadded:: 5.1.0 + .. versionchanged:: 5.5.1 added FreeBSD support. Memory ------ diff --git a/psutil/__init__.py b/psutil/__init__.py index 5a5720d7a..3548cdcce 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -220,7 +220,7 @@ ] __all__.extend(_psplatform.__extra__all__) __author__ = "Giampaolo Rodola'" -__version__ = "5.5.0" +__version__ = "5.5.1" version_info = tuple([int(num) for num in __version__.split('.')]) AF_LINK = _psplatform.AF_LINK POWER_TIME_UNLIMITED = _common.POWER_TIME_UNLIMITED diff --git a/psutil/_psbsd.py b/psutil/_psbsd.py index d3ce7b5cc..1dc723128 100644 --- a/psutil/_psbsd.py +++ b/psutil/_psbsd.py @@ -456,9 +456,9 @@ def sensors_temperatures(): return ret def cpu_freq(): - """ - Return frequency metrics for CPUs. Currently, only CPU 0 is supported - by FreeBSD, all other cores match the frequency of CPU 0. + """Return frequency metrics for CPUs. As of Dec 2018 only + CPU 0 appears to be supported by FreeBSD and all other cores + match the frequency of CPU 0. """ ret = [] num_cpus = cpu_count_logical() @@ -467,17 +467,15 @@ def cpu_freq(): current, available_freq = cext.cpu_frequency(cpu) except NotImplementedError: continue - min_freq = None - max_freq = None if available_freq: try: min_freq = int(available_freq.split(" ")[-1].split("/")[0]) except(IndexError, ValueError): - pass + min_freq = None try: max_freq = int(available_freq.split(" ")[0].split("/")[0]) except(IndexError, ValueError): - pass + max_freq = None ret.append(_common.scpufreq(current, min_freq, max_freq)) return ret diff --git a/psutil/arch/freebsd/specific.c b/psutil/arch/freebsd/specific.c index 70dc7f2ce..93bf51ff2 100644 --- a/psutil/arch/freebsd/specific.c +++ b/psutil/arch/freebsd/specific.c @@ -1049,7 +1049,9 @@ psutil_sensors_cpu_temperature(PyObject *self, PyObject *args) { /* - * Return frequency information of a given CPU + * Return frequency information of a given CPU. + * As of Dec 2018 only CPU 0 appears to be supported and all other + * cores match the frequency of CPU 0. */ PyObject * psutil_cpu_freq(PyObject *self, PyObject *args) { @@ -1061,12 +1063,14 @@ psutil_cpu_freq(PyObject *self, PyObject *args) { if (! PyArg_ParseTuple(args, "i", &core)) return NULL; + // https://www.unix.com/man-page/FreeBSD/4/cpufreq/ sprintf(sensor, "dev.cpu.%d.freq", core); if (sysctlbyname(sensor, ¤t, &size, NULL, 0)) goto error; size = sizeof(available_freq_levels); - // In case of failure, an empty string is returned + // https://www.unix.com/man-page/FreeBSD/4/cpufreq/ + // In case of failure, an empty string is returned. sprintf(sensor, "dev.cpu.%d.freq_levels", core); sysctlbyname(sensor, &available_freq_levels, &size, NULL, 0); @@ -1074,7 +1078,7 @@ psutil_cpu_freq(PyObject *self, PyObject *args) { error: if (errno == ENOENT) - PyErr_SetString(PyExc_NotImplementedError, "Unable to read frequency"); + PyErr_SetString(PyExc_NotImplementedError, "unable to read frequency"); else PyErr_SetFromErrno(PyExc_OSError); return NULL; diff --git a/psutil/tests/test_bsd.py b/psutil/tests/test_bsd.py index 87feb1473..34b66ca6b 100755 --- a/psutil/tests/test_bsd.py +++ b/psutil/tests/test_bsd.py @@ -250,10 +250,9 @@ def test_proc_cpu_times(self): if len(tested) != 2: raise RuntimeError("couldn't find lines match in procstat out") - # --- cpu_freq(); tests against sysctl def test_cpu_frequency_against_sysctl(self): # Currently only cpu 0 is frequency is supported in FreeBSD - # All other cores use the same frequency + # All other cores use the same frequency. sensor = "dev.cpu.0.freq" sysctl_result = int(sysctl(sensor)) self.assertEqual(psutil.cpu_freq().current, sysctl_result) @@ -262,7 +261,7 @@ def test_cpu_frequency_against_sysctl(self): sysctl_result = sysctl(sensor) # sysctl returns a string of the format: # / /... - # Ordered highest available to lowest available + # Ordered highest available to lowest available. max_freq = int(sysctl_result.split()[0].split("/")[0]) min_freq = int(sysctl_result.split()[-1].split("/")[0]) self.assertEqual(psutil.cpu_freq().max, max_freq) From 3ea94c1b8589891a8d1a5781f0445cb5080b7c3e Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 1 Dec 2018 19:29:19 +0100 Subject: [PATCH 161/182] make flake8 happy --- psutil/tests/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index eb49b9c2b..c6fc8cc23 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -635,7 +635,7 @@ def wrapper(*args, **kwargs): for _ in self: try: return fun(*args, **kwargs) - except self.exception as _: + except self.exception as _: # NOQA exc = _ if self.logfun is not None: self.logfun(exc) From b2dbcbc407920a39a0e0087ef3f507e6dab747cb Mon Sep 17 00:00:00 2001 From: EccoTheFlintstone <32797240+EccoTheFlintstone@users.noreply.github.com> Date: Mon, 3 Dec 2018 07:51:01 -0500 Subject: [PATCH 162/182] fix ionice set not working on windows x64 due to LENGTH_MISMATCH (#1368) Fix Process().ionice(0/1/2) length mismatch on Windows --- psutil/_psutil_windows.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/psutil/_psutil_windows.c b/psutil/_psutil_windows.c index edb5996c4..29311992b 100644 --- a/psutil/_psutil_windows.c +++ b/psutil/_psutil_windows.c @@ -2098,7 +2098,7 @@ static PyObject * psutil_proc_io_priority_get(PyObject *self, PyObject *args) { long pid; HANDLE hProcess; - PULONG IoPriority; + DWORD IoPriority; _NtQueryInformationProcess NtQueryInformationProcess = (_NtQueryInformationProcess)GetProcAddress( @@ -2114,7 +2114,7 @@ psutil_proc_io_priority_get(PyObject *self, PyObject *args) { hProcess, ProcessIoPriority, &IoPriority, - sizeof(ULONG), + sizeof(DWORD), NULL ); CloseHandle(hProcess); @@ -2128,8 +2128,9 @@ psutil_proc_io_priority_get(PyObject *self, PyObject *args) { static PyObject * psutil_proc_io_priority_set(PyObject *self, PyObject *args) { long pid; - int prio; + DWORD prio; HANDLE hProcess; + DWORD dwDesiredAccess = PROCESS_QUERY_INFORMATION | PROCESS_SET_INFORMATION; _NtSetInformationProcess NtSetInformationProcess = (_NtSetInformationProcess)GetProcAddress( @@ -2143,7 +2144,7 @@ psutil_proc_io_priority_set(PyObject *self, PyObject *args) { if (! PyArg_ParseTuple(args, "li", &pid, &prio)) return NULL; - hProcess = psutil_handle_from_pid_waccess(pid, PROCESS_ALL_ACCESS); + hProcess = psutil_handle_from_pid_waccess(pid, dwDesiredAccess); if (hProcess == NULL) return NULL; @@ -2151,7 +2152,7 @@ psutil_proc_io_priority_set(PyObject *self, PyObject *args) { hProcess, ProcessIoPriority, (PVOID)&prio, - sizeof((PVOID)prio) + sizeof(DWORD) ); CloseHandle(hProcess); From 9f44b6aa4b759c9c4f41dc9cd7df135a013f27ed Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 3 Dec 2018 13:53:02 +0100 Subject: [PATCH 163/182] give CREDITS to @EccoTheFlintstone for #1368 --- CREDITS | 4 ++++ HISTORY.rst | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/CREDITS b/CREDITS index 30d9006f9..f238f85ba 100644 --- a/CREDITS +++ b/CREDITS @@ -567,3 +567,7 @@ I: 1320 N: Koen Kooi W: https://github.com/koenkooi I: 1360 + +N: EccoTheFlintstone +W: https://github.com/EccoTheFlintstone +I: 1368 diff --git a/HISTORY.rst b/HISTORY.rst index 7cfe0f191..1adea0962 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -7,6 +7,11 @@ - 1352_: [FreeBSD] added support for CPU frequency. (patch by Alex Manuskin) +**Bug fixes** + +- 1368_: [Windows] fix psutil.Process().ionice(...) mismatch. (patch by + EccoTheFlintstone) + 5.5.0 ===== From c0f6b1d6514bad029995305e68dd127206e82864 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 5 Dec 2018 11:32:31 +0100 Subject: [PATCH 164/182] sort HISTORY --- HISTORY.rst | 18 ++++++------------ psutil/__init__.py | 2 +- 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 1adea0962..7a26f50e3 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,28 +1,22 @@ *Bug tracker at https://github.com/giampaolo/psutil/issues* -5.5.1 +5.5.0 ===== +XXXX-XX-XX + **Enhancements** +- 1350_: [FreeBSD] added support for sensors_temperatures(). (patch by Alex + Manuskin) - 1352_: [FreeBSD] added support for CPU frequency. (patch by Alex Manuskin) **Bug fixes** +- 1354_: [Linux] disk_io_counters() fails on Linux kernel 4.18+. - 1368_: [Windows] fix psutil.Process().ionice(...) mismatch. (patch by EccoTheFlintstone) -5.5.0 -===== - -**Enhancements** - -- 1350_: [FreeBSD] added support for sensors_temperatures(). - -**Bug fixes** - -- 1354_: [Linux] disk_io_counters() fails on Linux kernel 4.18+. - 5.4.8 ===== diff --git a/psutil/__init__.py b/psutil/__init__.py index 3548cdcce..5a5720d7a 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -220,7 +220,7 @@ ] __all__.extend(_psplatform.__extra__all__) __author__ = "Giampaolo Rodola'" -__version__ = "5.5.1" +__version__ = "5.5.0" version_info = tuple([int(num) for num in __version__.split('.')]) AF_LINK = _psplatform.AF_LINK POWER_TIME_UNLIMITED = _common.POWER_TIME_UNLIMITED From 5398c48047d424af97644879fb4eaa7aad432f58 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 8 Dec 2018 12:32:31 +0100 Subject: [PATCH 165/182] #1111 make Process.oneshot() thread-safe --- HISTORY.rst | 1 + psutil/__init__.py | 69 ++++++++++++++++++++++++---------------------- 2 files changed, 37 insertions(+), 33 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 7a26f50e3..e66f5b49c 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -13,6 +13,7 @@ XXXX-XX-XX **Bug fixes** +- 1111_: Process.oneshot() is now thread safe. - 1354_: [Linux] disk_io_counters() fails on Linux kernel 4.18+. - 1368_: [Windows] fix psutil.Process().ionice(...) mismatch. (patch by EccoTheFlintstone) diff --git a/psutil/__init__.py b/psutil/__init__.py index 5a5720d7a..0eb197992 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -31,6 +31,7 @@ import signal import subprocess import sys +import threading import time try: import pwd @@ -360,6 +361,7 @@ def _init(self, pid, _ignore_nsp=False): self._proc = _psplatform.Process(pid) self._last_sys_cpu_times = None self._last_proc_cpu_times = None + self._lock = threading.RLock() # cache creation time for later use in is_running() method try: self.create_time() @@ -456,40 +458,41 @@ def oneshot(self): ... >>> """ - if self._oneshot_inctx: - # NOOP: this covers the use case where the user enters the - # context twice. Since as_dict() internally uses oneshot() - # I expect that the code below will be a pretty common - # "mistake" that the user will make, so let's guard - # against that: - # - # >>> with p.oneshot(): - # ... p.as_dict() - # ... - yield - else: - self._oneshot_inctx = True - try: - # cached in case cpu_percent() is used - self.cpu_times.cache_activate() - # cached in case memory_percent() is used - self.memory_info.cache_activate() - # cached in case parent() is used - self.ppid.cache_activate() - # cached in case username() is used - if POSIX: - self.uids.cache_activate() - # specific implementation cache - self._proc.oneshot_enter() + with self._lock: + if self._oneshot_inctx: + # NOOP: this covers the use case where the user enters the + # context twice. Since as_dict() internally uses oneshot() + # I expect that the code below will be a pretty common + # "mistake" that the user will make, so let's guard + # against that: + # + # >>> with p.oneshot(): + # ... p.as_dict() + # ... yield - finally: - self.cpu_times.cache_deactivate() - self.memory_info.cache_deactivate() - self.ppid.cache_deactivate() - if POSIX: - self.uids.cache_deactivate() - self._proc.oneshot_exit() - self._oneshot_inctx = False + else: + self._oneshot_inctx = True + try: + # cached in case cpu_percent() is used + self.cpu_times.cache_activate() + # cached in case memory_percent() is used + self.memory_info.cache_activate() + # cached in case parent() is used + self.ppid.cache_activate() + # cached in case username() is used + if POSIX: + self.uids.cache_activate() + # specific implementation cache + self._proc.oneshot_enter() + yield + finally: + self.cpu_times.cache_deactivate() + self.memory_info.cache_deactivate() + self.ppid.cache_deactivate() + if POSIX: + self.uids.cache_deactivate() + self._proc.oneshot_exit() + self._oneshot_inctx = False def as_dict(self, attrs=None, ad_value=None): """Utility method returning process information as a From fca240a81b78de019d69bfc981f38c7fd859bb6d Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 8 Dec 2018 12:40:20 +0100 Subject: [PATCH 166/182] revert 5398c48047d424af97644879fb4eaa7aad432f58; let's do it in a separate branch --- psutil/__init__.py | 69 ++++++++++++++++++++++------------------------ 1 file changed, 33 insertions(+), 36 deletions(-) diff --git a/psutil/__init__.py b/psutil/__init__.py index 0eb197992..5a5720d7a 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -31,7 +31,6 @@ import signal import subprocess import sys -import threading import time try: import pwd @@ -361,7 +360,6 @@ def _init(self, pid, _ignore_nsp=False): self._proc = _psplatform.Process(pid) self._last_sys_cpu_times = None self._last_proc_cpu_times = None - self._lock = threading.RLock() # cache creation time for later use in is_running() method try: self.create_time() @@ -458,41 +456,40 @@ def oneshot(self): ... >>> """ - with self._lock: - if self._oneshot_inctx: - # NOOP: this covers the use case where the user enters the - # context twice. Since as_dict() internally uses oneshot() - # I expect that the code below will be a pretty common - # "mistake" that the user will make, so let's guard - # against that: - # - # >>> with p.oneshot(): - # ... p.as_dict() - # ... + if self._oneshot_inctx: + # NOOP: this covers the use case where the user enters the + # context twice. Since as_dict() internally uses oneshot() + # I expect that the code below will be a pretty common + # "mistake" that the user will make, so let's guard + # against that: + # + # >>> with p.oneshot(): + # ... p.as_dict() + # ... + yield + else: + self._oneshot_inctx = True + try: + # cached in case cpu_percent() is used + self.cpu_times.cache_activate() + # cached in case memory_percent() is used + self.memory_info.cache_activate() + # cached in case parent() is used + self.ppid.cache_activate() + # cached in case username() is used + if POSIX: + self.uids.cache_activate() + # specific implementation cache + self._proc.oneshot_enter() yield - else: - self._oneshot_inctx = True - try: - # cached in case cpu_percent() is used - self.cpu_times.cache_activate() - # cached in case memory_percent() is used - self.memory_info.cache_activate() - # cached in case parent() is used - self.ppid.cache_activate() - # cached in case username() is used - if POSIX: - self.uids.cache_activate() - # specific implementation cache - self._proc.oneshot_enter() - yield - finally: - self.cpu_times.cache_deactivate() - self.memory_info.cache_deactivate() - self.ppid.cache_deactivate() - if POSIX: - self.uids.cache_deactivate() - self._proc.oneshot_exit() - self._oneshot_inctx = False + finally: + self.cpu_times.cache_deactivate() + self.memory_info.cache_deactivate() + self.ppid.cache_deactivate() + if POSIX: + self.uids.cache_deactivate() + self._proc.oneshot_exit() + self._oneshot_inctx = False def as_dict(self, attrs=None, ad_value=None): """Utility method returning process information as a From 10f780b7c2c0bb63417360891662680a39465140 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 8 Dec 2018 12:42:02 +0100 Subject: [PATCH 167/182] update HISTORY --- HISTORY.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index e66f5b49c..7a26f50e3 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -13,7 +13,6 @@ XXXX-XX-XX **Bug fixes** -- 1111_: Process.oneshot() is now thread safe. - 1354_: [Linux] disk_io_counters() fails on Linux kernel 4.18+. - 1368_: [Windows] fix psutil.Process().ionice(...) mismatch. (patch by EccoTheFlintstone) From 0cc8d7b5e206203541193a8e2120b2689a7a5893 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 8 Dec 2018 08:26:39 -0800 Subject: [PATCH 168/182] (Windows) use PROCESS_QUERY_LIMITED_INFORMATION access rights (#1376) #1376 / Windows / OpenProcess - use PROCESS_QUERY_LIMITED_INFORMATION wherever possible. This results in less AccessDenied exceptions being thrown for system processes. --- psutil/_psutil_windows.c | 44 ++++++++++++++++-------------- psutil/arch/windows/process_info.c | 17 ++---------- psutil/arch/windows/process_info.h | 3 +- 3 files changed, 28 insertions(+), 36 deletions(-) diff --git a/psutil/_psutil_windows.c b/psutil/_psutil_windows.c index 29311992b..f3979de62 100644 --- a/psutil/_psutil_windows.c +++ b/psutil/_psutil_windows.c @@ -492,7 +492,8 @@ psutil_proc_cpu_times(PyObject *self, PyObject *args) { if (! PyArg_ParseTuple(args, "l", &pid)) return NULL; - hProcess = psutil_handle_from_pid(pid); + hProcess = psutil_handle_from_pid(pid, PROCESS_QUERY_LIMITED_INFORMATION); + if (hProcess == NULL) return NULL; if (! GetProcessTimes(hProcess, &ftCreate, &ftExit, &ftKernel, &ftUser)) { @@ -546,7 +547,7 @@ psutil_proc_create_time(PyObject *self, PyObject *args) { if (0 == pid || 4 == pid) return psutil_boot_time(NULL, NULL); - hProcess = psutil_handle_from_pid(pid); + hProcess = psutil_handle_from_pid(pid, PROCESS_QUERY_LIMITED_INFORMATION); if (hProcess == NULL) return NULL; if (! GetProcessTimes(hProcess, &ftCreate, &ftExit, &ftKernel, &ftUser)) { @@ -756,7 +757,7 @@ psutil_proc_exe(PyObject *self, PyObject *args) { if (! PyArg_ParseTuple(args, "l", &pid)) return NULL; - hProcess = psutil_handle_from_pid_waccess(pid, PROCESS_QUERY_INFORMATION); + hProcess = psutil_handle_from_pid(pid, PROCESS_QUERY_LIMITED_INFORMATION); if (NULL == hProcess) return NULL; if (GetProcessImageFileNameW(hProcess, exe, MAX_PATH) == 0) { @@ -824,7 +825,7 @@ psutil_proc_memory_info(PyObject *self, PyObject *args) { if (! PyArg_ParseTuple(args, "l", &pid)) return NULL; - hProcess = psutil_handle_from_pid(pid); + hProcess = psutil_handle_from_pid(pid, PROCESS_QUERY_LIMITED_INFORMATION); if (NULL == hProcess) return NULL; @@ -892,6 +893,8 @@ psutil_proc_memory_uss(PyObject *self, PyObject *args) size_t private_pages; size_t i; DWORD info_array_size; + // needed by QueryWorkingSet + DWORD access = PROCESS_QUERY_INFORMATION | PROCESS_VM_READ; PSAPI_WORKING_SET_INFORMATION* info_array; SYSTEM_INFO system_info; PyObject* py_result = NULL; @@ -900,7 +903,8 @@ psutil_proc_memory_uss(PyObject *self, PyObject *args) if (! PyArg_ParseTuple(args, "l", &pid)) return NULL; - proc = psutil_handle_from_pid(pid); + + proc = psutil_handle_from_pid(pid, access); if (proc == NULL) return NULL; @@ -1350,7 +1354,7 @@ psutil_proc_open_files(PyObject *self, PyObject *args) { if (! PyArg_ParseTuple(args, "l", &pid)) return NULL; - processHandle = psutil_handle_from_pid_waccess(pid, access); + processHandle = psutil_handle_from_pid(pid, access); if (processHandle == NULL) return NULL; py_retlist = psutil_get_open_files(pid, processHandle); @@ -1412,8 +1416,7 @@ psutil_proc_username(PyObject *self, PyObject *args) { if (! PyArg_ParseTuple(args, "l", &pid)) return NULL; - processHandle = psutil_handle_from_pid_waccess( - pid, PROCESS_QUERY_INFORMATION); + processHandle = psutil_handle_from_pid(pid, PROCESS_QUERY_INFORMATION); if (processHandle == NULL) return NULL; @@ -2055,7 +2058,7 @@ psutil_proc_priority_get(PyObject *self, PyObject *args) { if (! PyArg_ParseTuple(args, "l", &pid)) return NULL; - hProcess = psutil_handle_from_pid(pid); + hProcess = psutil_handle_from_pid(pid, PROCESS_QUERY_LIMITED_INFORMATION); if (hProcess == NULL) return NULL; priority = GetPriorityClass(hProcess); @@ -2079,7 +2082,7 @@ psutil_proc_priority_set(PyObject *self, PyObject *args) { if (! PyArg_ParseTuple(args, "li", &pid, &priority)) return NULL; - hProcess = psutil_handle_from_pid_waccess(pid, access); + hProcess = psutil_handle_from_pid(pid, access); if (hProcess == NULL) return NULL; retval = SetPriorityClass(hProcess, priority); @@ -2106,7 +2109,7 @@ psutil_proc_io_priority_get(PyObject *self, PyObject *args) { if (! PyArg_ParseTuple(args, "l", &pid)) return NULL; - hProcess = psutil_handle_from_pid(pid); + hProcess = psutil_handle_from_pid(pid, PROCESS_QUERY_LIMITED_INFORMATION); if (hProcess == NULL) return NULL; @@ -2130,7 +2133,7 @@ psutil_proc_io_priority_set(PyObject *self, PyObject *args) { long pid; DWORD prio; HANDLE hProcess; - DWORD dwDesiredAccess = PROCESS_QUERY_INFORMATION | PROCESS_SET_INFORMATION; + DWORD access = PROCESS_QUERY_INFORMATION | PROCESS_SET_INFORMATION; _NtSetInformationProcess NtSetInformationProcess = (_NtSetInformationProcess)GetProcAddress( @@ -2144,7 +2147,7 @@ psutil_proc_io_priority_set(PyObject *self, PyObject *args) { if (! PyArg_ParseTuple(args, "li", &pid, &prio)) return NULL; - hProcess = psutil_handle_from_pid_waccess(pid, dwDesiredAccess); + hProcess = psutil_handle_from_pid(pid, access); if (hProcess == NULL) return NULL; @@ -2172,7 +2175,7 @@ psutil_proc_io_counters(PyObject *self, PyObject *args) { if (! PyArg_ParseTuple(args, "l", &pid)) return NULL; - hProcess = psutil_handle_from_pid(pid); + hProcess = psutil_handle_from_pid(pid, PROCESS_QUERY_LIMITED_INFORMATION); if (NULL == hProcess) return NULL; if (! GetProcessIoCounters(hProcess, &IoCounters)) { @@ -2202,7 +2205,7 @@ psutil_proc_cpu_affinity_get(PyObject *self, PyObject *args) { if (! PyArg_ParseTuple(args, "l", &pid)) return NULL; - hProcess = psutil_handle_from_pid(pid); + hProcess = psutil_handle_from_pid(pid, PROCESS_QUERY_LIMITED_INFORMATION); if (hProcess == NULL) { return NULL; } @@ -2227,8 +2230,7 @@ static PyObject * psutil_proc_cpu_affinity_set(PyObject *self, PyObject *args) { DWORD pid; HANDLE hProcess; - DWORD dwDesiredAccess = \ - PROCESS_QUERY_INFORMATION | PROCESS_SET_INFORMATION; + DWORD access = PROCESS_QUERY_INFORMATION | PROCESS_SET_INFORMATION; DWORD_PTR mask; #ifdef _WIN64 @@ -2239,7 +2241,7 @@ psutil_proc_cpu_affinity_set(PyObject *self, PyObject *args) { { return NULL; } - hProcess = psutil_handle_from_pid_waccess(pid, dwDesiredAccess); + hProcess = psutil_handle_from_pid(pid, access); if (hProcess == NULL) return NULL; @@ -2877,7 +2879,7 @@ psutil_proc_num_handles(PyObject *self, PyObject *args) { if (! PyArg_ParseTuple(args, "l", &pid)) return NULL; - hProcess = psutil_handle_from_pid(pid); + hProcess = psutil_handle_from_pid(pid, PROCESS_QUERY_LIMITED_INFORMATION); if (NULL == hProcess) return NULL; if (! GetProcessHandleCount(hProcess, &handleCount)) { @@ -3025,6 +3027,8 @@ psutil_proc_memory_maps(PyObject *self, PyObject *args) { WCHAR mappedFileName[MAX_PATH]; SYSTEM_INFO system_info; LPVOID maxAddr; + // required by GetMappedFileNameW + DWORD access = PROCESS_QUERY_INFORMATION | PROCESS_VM_READ; PyObject *py_retlist = PyList_New(0); PyObject *py_tuple = NULL; PyObject *py_str = NULL; @@ -3033,7 +3037,7 @@ psutil_proc_memory_maps(PyObject *self, PyObject *args) { return NULL; if (! PyArg_ParseTuple(args, "l", &pid)) goto error; - hProcess = psutil_handle_from_pid(pid); + hProcess = psutil_handle_from_pid(pid, access); if (NULL == hProcess) goto error; diff --git a/psutil/arch/windows/process_info.c b/psutil/arch/windows/process_info.c index ffd3c80ef..628c01abd 100644 --- a/psutil/arch/windows/process_info.c +++ b/psutil/arch/windows/process_info.c @@ -272,7 +272,7 @@ psutil_check_phandle(HANDLE hProcess, DWORD pid) { * Return a process handle or NULL. */ HANDLE -psutil_handle_from_pid_waccess(DWORD pid, DWORD dwDesiredAccess) { +psutil_handle_from_pid(DWORD pid, DWORD dwDesiredAccess) { HANDLE hProcess; if (pid == 0) { @@ -285,18 +285,6 @@ psutil_handle_from_pid_waccess(DWORD pid, DWORD dwDesiredAccess) { } -/* - * Same as psutil_handle_from_pid_waccess but implicitly uses - * PROCESS_QUERY_INFORMATION | PROCESS_VM_READ as dwDesiredAccess - * parameter for OpenProcess. - */ -HANDLE -psutil_handle_from_pid(DWORD pid) { - DWORD dwDesiredAccess = PROCESS_QUERY_INFORMATION | PROCESS_VM_READ; - return psutil_handle_from_pid_waccess(pid, dwDesiredAccess); -} - - DWORD * psutil_get_pids(DWORD *numberOfReturnedPIDs) { // Win32 SDK says the only way to know if our process array @@ -553,8 +541,9 @@ static int psutil_get_process_data(long pid, BOOL weAreWow64; BOOL theyAreWow64; #endif + DWORD access = PROCESS_QUERY_INFORMATION | PROCESS_VM_READ; - hProcess = psutil_handle_from_pid(pid); + hProcess = psutil_handle_from_pid(pid, access); if (hProcess == NULL) return -1; diff --git a/psutil/arch/windows/process_info.h b/psutil/arch/windows/process_info.h index a2f70c2b9..f85c1efdf 100644 --- a/psutil/arch/windows/process_info.h +++ b/psutil/arch/windows/process_info.h @@ -17,8 +17,7 @@ DWORD* psutil_get_pids(DWORD *numberOfReturnedPIDs); -HANDLE psutil_handle_from_pid(DWORD pid); -HANDLE psutil_handle_from_pid_waccess(DWORD pid, DWORD dwDesiredAccess); +HANDLE psutil_handle_from_pid(DWORD pid, DWORD dwDesiredAccess); int psutil_pid_is_running(DWORD pid); int psutil_get_proc_info(DWORD pid, PSYSTEM_PROCESS_INFORMATION *retProcess, PVOID *retBuffer); From f88ca356b7fe43efc04e2610cf0190d5aeb22188 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 8 Dec 2018 17:46:14 +0100 Subject: [PATCH 169/182] update HISTORY --- HISTORY.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index 7a26f50e3..1b4e1c96c 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -16,6 +16,9 @@ XXXX-XX-XX - 1354_: [Linux] disk_io_counters() fails on Linux kernel 4.18+. - 1368_: [Windows] fix psutil.Process().ionice(...) mismatch. (patch by EccoTheFlintstone) +- 1376_: [Windows] OpenProcess() now uses PROCESS_QUERY_LIMITED_INFORMATION + access rights wherever possible, resulting in less AccessDenied exceptions + being thrown for system processes. 5.4.8 ===== From c9988242c7138802e48ecd09837e95a06096aae6 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 8 Dec 2018 09:18:23 -0800 Subject: [PATCH 170/182] fix #1370: improper usage of CloseHandle() may lead to override the original error code resulting in raising a wrong exception --- HISTORY.rst | 2 + psutil/_psutil_windows.c | 76 ++++++++++++++++++++---------- psutil/arch/windows/process_info.c | 2 +- 3 files changed, 55 insertions(+), 25 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 1b4e1c96c..366485715 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -16,6 +16,8 @@ XXXX-XX-XX - 1354_: [Linux] disk_io_counters() fails on Linux kernel 4.18+. - 1368_: [Windows] fix psutil.Process().ionice(...) mismatch. (patch by EccoTheFlintstone) +- 1370_: [Windows] improper usage of CloseHandle() may lead to override the + original error code when raising an exception. - 1376_: [Windows] OpenProcess() now uses PROCESS_QUERY_LIMITED_INFORMATION access rights wherever possible, resulting in less AccessDenied exceptions being thrown for system processes. diff --git a/psutil/_psutil_windows.c b/psutil/_psutil_windows.c index f3979de62..cbd7ea8cb 100644 --- a/psutil/_psutil_windows.c +++ b/psutil/_psutil_windows.c @@ -399,8 +399,8 @@ psutil_proc_kill(PyObject *self, PyObject *args) { err = GetLastError(); // See: https://github.com/giampaolo/psutil/issues/1099 if (err != ERROR_ACCESS_DENIED) { - CloseHandle(hProcess); PyErr_SetFromWindowsErr(err); + CloseHandle(hProcess); return NULL; } } @@ -445,21 +445,21 @@ psutil_proc_wait(PyObject *self, PyObject *args) { // handle return code if (retVal == WAIT_FAILED) { - CloseHandle(hProcess); PyErr_SetFromWindowsErr(0); + CloseHandle(hProcess); return NULL; } if (retVal == WAIT_TIMEOUT) { - CloseHandle(hProcess); PyErr_SetString(TimeoutExpired, "WaitForSingleObject() returned WAIT_TIMEOUT"); + CloseHandle(hProcess); return NULL; } if (retVal == WAIT_ABANDONED) { psutil_debug("WaitForSingleObject() -> WAIT_ABANDONED"); - CloseHandle(hProcess); PyErr_SetString(TimeoutAbandoned, "WaitForSingleObject() returned WAIT_ABANDONED"); + CloseHandle(hProcess); return NULL; } @@ -467,9 +467,11 @@ psutil_proc_wait(PyObject *self, PyObject *args) { // process is gone so we can get its process exit code. The PID // may still stick around though but we'll handle that from Python. if (GetExitCodeProcess(hProcess, &ExitCode) == 0) { + PyErr_SetFromWindowsErr(0); CloseHandle(hProcess); - return PyErr_SetFromWindowsErr(GetLastError()); + return NULL; } + CloseHandle(hProcess); #if PY_MAJOR_VERSION >= 3 @@ -497,15 +499,16 @@ psutil_proc_cpu_times(PyObject *self, PyObject *args) { if (hProcess == NULL) return NULL; if (! GetProcessTimes(hProcess, &ftCreate, &ftExit, &ftKernel, &ftUser)) { - CloseHandle(hProcess); if (GetLastError() == ERROR_ACCESS_DENIED) { // usually means the process has died so we throw a NoSuchProcess // here - return NoSuchProcess(""); + NoSuchProcess(""); } else { - return PyErr_SetFromWindowsErr(0); + PyErr_SetFromWindowsErr(0); } + CloseHandle(hProcess); + return NULL; } CloseHandle(hProcess); @@ -551,15 +554,16 @@ psutil_proc_create_time(PyObject *self, PyObject *args) { if (hProcess == NULL) return NULL; if (! GetProcessTimes(hProcess, &ftCreate, &ftExit, &ftKernel, &ftUser)) { - CloseHandle(hProcess); if (GetLastError() == ERROR_ACCESS_DENIED) { // usually means the process has died so we throw a // NoSuchProcess here - return NoSuchProcess(""); + NoSuchProcess(""); } else { - return PyErr_SetFromWindowsErr(0); + PyErr_SetFromWindowsErr(0); } + CloseHandle(hProcess); + return NULL; } CloseHandle(hProcess); @@ -761,8 +765,9 @@ psutil_proc_exe(PyObject *self, PyObject *args) { if (NULL == hProcess) return NULL; if (GetProcessImageFileNameW(hProcess, exe, MAX_PATH) == 0) { + PyErr_SetFromWindowsErr(0); CloseHandle(hProcess); - return PyErr_SetFromWindowsErr(0); + return NULL; } CloseHandle(hProcess); return PyUnicode_FromWideChar(exe, wcslen(exe)); @@ -790,8 +795,9 @@ psutil_proc_name(PyObject *self, PyObject *args) { pentry.dwSize = sizeof(PROCESSENTRY32W); ok = Process32FirstW(hSnapShot, &pentry); if (! ok) { + PyErr_SetFromWindowsErr(0); CloseHandle(hSnapShot); - return PyErr_SetFromWindowsErr(0); + return NULL; } while (ok) { if (pentry.th32ProcessID == pid) { @@ -831,8 +837,9 @@ psutil_proc_memory_info(PyObject *self, PyObject *args) { if (! GetProcessMemoryInfo(hProcess, (PPROCESS_MEMORY_COUNTERS)&cnt, sizeof(cnt))) { + PyErr_SetFromWindowsErr(0); CloseHandle(hProcess); - return PyErr_SetFromWindowsErr(0); + return NULL; } #if (_WIN32_WINNT >= 0x0501) // Windows XP with SP2 @@ -1357,10 +1364,15 @@ psutil_proc_open_files(PyObject *self, PyObject *args) { processHandle = psutil_handle_from_pid(pid, access); if (processHandle == NULL) return NULL; + py_retlist = psutil_get_open_files(pid, processHandle); + if (py_retlist == NULL) { + PyErr_SetFromWindowsErr(0); + CloseHandle(processHandle); + return NULL; + } + CloseHandle(processHandle); - if (py_retlist == NULL) - return PyErr_SetFromWindowsErr(0); return py_retlist; } @@ -2058,13 +2070,18 @@ psutil_proc_priority_get(PyObject *self, PyObject *args) { if (! PyArg_ParseTuple(args, "l", &pid)) return NULL; + hProcess = psutil_handle_from_pid(pid, PROCESS_QUERY_LIMITED_INFORMATION); if (hProcess == NULL) return NULL; + priority = GetPriorityClass(hProcess); + if (priority == 0) { + PyErr_SetFromWindowsErr(0); + CloseHandle(hProcess); + return NULL; + } CloseHandle(hProcess); - if (priority == 0) - return PyErr_SetFromWindowsErr(0); return Py_BuildValue("i", priority); } @@ -2085,10 +2102,15 @@ psutil_proc_priority_set(PyObject *self, PyObject *args) { hProcess = psutil_handle_from_pid(pid, access); if (hProcess == NULL) return NULL; + retval = SetPriorityClass(hProcess, priority); + if (retval == 0) { + PyErr_SetFromWindowsErr(0); + CloseHandle(hProcess); + return NULL; + } + CloseHandle(hProcess); - if (retval == 0) - return PyErr_SetFromWindowsErr(0); Py_RETURN_NONE; } @@ -2178,10 +2200,13 @@ psutil_proc_io_counters(PyObject *self, PyObject *args) { hProcess = psutil_handle_from_pid(pid, PROCESS_QUERY_LIMITED_INFORMATION); if (NULL == hProcess) return NULL; + if (! GetProcessIoCounters(hProcess, &IoCounters)) { + PyErr_SetFromWindowsErr(0); CloseHandle(hProcess); - return PyErr_SetFromWindowsErr(0); + return NULL; } + CloseHandle(hProcess); return Py_BuildValue("(KKKKKK)", IoCounters.ReadOperationCount, @@ -2210,8 +2235,9 @@ psutil_proc_cpu_affinity_get(PyObject *self, PyObject *args) { return NULL; } if (GetProcessAffinityMask(hProcess, &proc_mask, &system_mask) == 0) { + PyErr_SetFromWindowsErr(0); CloseHandle(hProcess); - return PyErr_SetFromWindowsErr(0); + return NULL; } CloseHandle(hProcess); @@ -2246,8 +2272,9 @@ psutil_proc_cpu_affinity_set(PyObject *self, PyObject *args) { return NULL; if (SetProcessAffinityMask(hProcess, mask) == 0) { + PyErr_SetFromWindowsErr(0); CloseHandle(hProcess); - return PyErr_SetFromWindowsErr(0); + return NULL; } CloseHandle(hProcess); @@ -2883,8 +2910,9 @@ psutil_proc_num_handles(PyObject *self, PyObject *args) { if (NULL == hProcess) return NULL; if (! GetProcessHandleCount(hProcess, &handleCount)) { + PyErr_SetFromWindowsErr(0); CloseHandle(hProcess); - return PyErr_SetFromWindowsErr(0); + return NULL; } CloseHandle(hProcess); return Py_BuildValue("k", handleCount); diff --git a/psutil/arch/windows/process_info.c b/psutil/arch/windows/process_info.c index 628c01abd..d83b71501 100644 --- a/psutil/arch/windows/process_info.c +++ b/psutil/arch/windows/process_info.c @@ -422,7 +422,7 @@ psutil_pid_is_running(DWORD pid) { return 1; } else { - PyErr_SetFromWindowsErr(0); + PyErr_SetFromWindowsErr(err); return -1; } } From 62410eb927d20003271fb0c5d66a21bf59f38628 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 8 Dec 2018 18:37:27 +0100 Subject: [PATCH 171/182] enforce lack of support for Win XP --- psutil/_pswindows.py | 3 +-- setup.py | 9 ++++----- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/psutil/_pswindows.py b/psutil/_pswindows.py index b938d42ff..2bc9c9dd0 100644 --- a/psutil/_pswindows.py +++ b/psutil/_pswindows.py @@ -27,8 +27,7 @@ # but if we get here it means this this was a wheel (or exe). msg = "this Windows version is too old (< Windows Vista); " msg += "psutil 3.4.2 is the latest version which supports Windows " - msg += "2000, XP and 2003 server; it may be possible that psutil " - msg += "will work if compiled from sources though" + msg += "2000, XP and 2003 server" raise RuntimeError(msg) else: raise diff --git a/setup.py b/setup.py index 28cde6f12..e28532369 100755 --- a/setup.py +++ b/setup.py @@ -109,11 +109,10 @@ def get_winver(): return '0x0%s' % ((maj * 100) + min) if sys.getwindowsversion()[0] < 6: - msg = "warning: Windows versions < Vista are no longer supported or " - msg = "maintained; latest official supported version is psutil 3.4.2; " - msg += "psutil may still be installed from sources if you have " - msg += "Visual Studio and may also (kind of) work though" - warnings.warn(msg, UserWarning) + msg = "this Windows version is too old (< Windows Vista); " + msg += "psutil 3.4.2 is the latest version which supports Windows " + msg += "2000, XP and 2003 server" + raise RuntimeError(msg) macros.append(("PSUTIL_WINDOWS", 1)) macros.extend([ From 790292d3902a51800117e9343be918ec1ddecbde Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 10 Dec 2018 14:47:00 +0100 Subject: [PATCH 172/182] #1376 Windows: check if variable is NULL before free()ing it --- HISTORY.rst | 2 ++ psutil/arch/windows/process_info.c | 13 ++++++++----- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 366485715..263a381f8 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -21,6 +21,8 @@ XXXX-XX-XX - 1376_: [Windows] OpenProcess() now uses PROCESS_QUERY_LIMITED_INFORMATION access rights wherever possible, resulting in less AccessDenied exceptions being thrown for system processes. +- 1376_: [Windows] check if variable is NULL before free()ing it. (patch by + EccoTheFlintstone) 5.4.8 ===== diff --git a/psutil/arch/windows/process_info.c b/psutil/arch/windows/process_info.c index d83b71501..b79aeb3e1 100644 --- a/psutil/arch/windows/process_info.c +++ b/psutil/arch/windows/process_info.c @@ -302,7 +302,8 @@ psutil_get_pids(DWORD *numberOfReturnedPIDs) { do { procArraySz += 1024; - free(procArray); + if (procArray != NULL) + free(procArray); procArrayByteSz = procArraySz * sizeof(DWORD); procArray = malloc(procArrayByteSz); if (procArray == NULL) { @@ -833,7 +834,8 @@ psutil_get_cmdline(long pid) { out: LocalFree(szArglist); - free(data); + if (data != NULL) + free(data); Py_XDECREF(py_unicode); Py_XDECREF(py_retlist); @@ -852,7 +854,8 @@ PyObject *psutil_get_cwd(long pid) { ret = PyUnicode_FromWideChar(data, wcslen(data)); out: - free(data); + if (data != NULL) + free(data); return ret; } @@ -873,8 +876,8 @@ PyObject *psutil_get_environ(long pid) { ret = PyUnicode_FromWideChar(data, size / 2); out: - free(data); - + if (data != NULL) + free(data); return ret; } From 314ab75295cbcdc6d8a12f89a08d702795593eca Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 10 Dec 2018 15:04:17 +0100 Subject: [PATCH 173/182] fix #1357: do not expose Process' memory_maps() and io_counters() methods if not supported by the kernel --- HISTORY.rst | 2 ++ psutil/_pslinux.py | 15 ++++----------- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 263a381f8..59bf9f678 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -14,6 +14,8 @@ XXXX-XX-XX **Bug fixes** - 1354_: [Linux] disk_io_counters() fails on Linux kernel 4.18+. +- 1357_: [Linux] Process' memory_maps() and io_counters() method are no longer + exposed if not supported by the kernel. - 1368_: [Windows] fix psutil.Process().ionice(...) mismatch. (patch by EccoTheFlintstone) - 1370_: [Windows] improper usage of CloseHandle() may lead to override the diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index b775d39ae..51cb20096 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -1642,6 +1642,7 @@ def terminal(self): except KeyError: return None + # May not be available on old kernels. if os.path.exists('/proc/%s/io' % os.getpid()): @wrap_exceptions def io_counters(self): @@ -1673,10 +1674,6 @@ def io_counters(self): except KeyError as err: raise ValueError("%r field was not found in %s; found fields " "are %r" % (err[0], fname, fields)) - else: - def io_counters(self): - raise NotImplementedError("couldn't find /proc/%s/io (kernel " - "too old?)" % self.pid) @wrap_exceptions def cpu_times(self): @@ -1767,6 +1764,9 @@ def memory_maps(self): """Return process's mapped memory regions as a list of named tuples. Fields are explained in 'man proc'; here is an updated (Apr 2012) version: http://goo.gl/fmebo + + /proc/{PID}/smaps does not exist on kernels < 2.6.14 or if + CONFIG_MMU kernel configuration option is not enabled. """ def get_blocks(lines, current_block): data = {} @@ -1827,13 +1827,6 @@ def get_blocks(lines, current_block): )) return ls - else: # pragma: no cover - def memory_maps(self): - raise NotImplementedError( - "/proc/%s/smaps does not exist on kernels < 2.6.14 or " - "if CONFIG_MMU kernel configuration option is not " - "enabled." % self.pid) - @wrap_exceptions def cwd(self): try: From d364283df3b7c42808704f3c7ef99536b1df784b Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 10 Dec 2018 16:38:24 +0100 Subject: [PATCH 174/182] Linux: refactor _parse_stat_file() and return a dict instead of a list (+ maintainability) --- psutil/_pslinux.py | 47 +++++++++++++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 17 deletions(-) diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index 51cb20096..880be2c82 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -1538,11 +1538,11 @@ def __init__(self, pid): @memoize_when_activated def _parse_stat_file(self): - """Parse /proc/{pid}/stat file. Return a list of fields where - process name is in position 0. + """Parse /proc/{pid}/stat file and return a dict with various + process info. Using "man proc" as a reference: where "man proc" refers to - position N, always substract 2 (e.g starttime pos 22 in - 'man proc' == pos 20 in the list returned here). + position N always substract 3 (e.g ppid position 4 in + 'man proc' == position 1 in here). The return value is cached in case oneshot() ctx manager is in use. """ @@ -1553,8 +1553,21 @@ def _parse_stat_file(self): # the first occurrence of "(" and the last occurence of ")". rpar = data.rfind(b')') name = data[data.find(b'(') + 1:rpar] - others = data[rpar + 2:].split() - return [name] + others + fields = data[rpar + 2:].split() + + ret = {} + ret['name'] = name + ret['status'] = fields[0] + ret['ppid'] = fields[1] + ret['ttynr'] = fields[4] + ret['utime'] = fields[11] + ret['stime'] = fields[12] + ret['children_utime'] = fields[13] + ret['children_stime'] = fields[14] + ret['create_time'] = fields[19] + ret['cpu_num'] = fields[36] + + return ret @memoize_when_activated def _read_status_file(self): @@ -1583,7 +1596,7 @@ def oneshot_exit(self): @wrap_exceptions def name(self): - name = self._parse_stat_file()[0] + name = self._parse_stat_file()['name'] if PY3: name = decode(name) # XXX - gets changed later and probably needs refactoring @@ -1635,7 +1648,7 @@ def environ(self): @wrap_exceptions def terminal(self): - tty_nr = int(self._parse_stat_file()[5]) + tty_nr = int(self._parse_stat_file()['ttynr']) tmap = _psposix.get_terminal_map() try: return tmap[tty_nr] @@ -1678,16 +1691,16 @@ def io_counters(self): @wrap_exceptions def cpu_times(self): values = self._parse_stat_file() - utime = float(values[12]) / CLOCK_TICKS - stime = float(values[13]) / CLOCK_TICKS - children_utime = float(values[14]) / CLOCK_TICKS - children_stime = float(values[15]) / CLOCK_TICKS + utime = float(values['utime']) / CLOCK_TICKS + stime = float(values['stime']) / CLOCK_TICKS + children_utime = float(values['children_utime']) / CLOCK_TICKS + children_stime = float(values['children_stime']) / CLOCK_TICKS return _common.pcputimes(utime, stime, children_utime, children_stime) @wrap_exceptions def cpu_num(self): """What CPU the process is on.""" - return int(self._parse_stat_file()[37]) + return int(self._parse_stat_file()['cpu_num']) @wrap_exceptions def wait(self, timeout=None): @@ -1695,14 +1708,14 @@ def wait(self, timeout=None): @wrap_exceptions def create_time(self): - values = self._parse_stat_file() + ctime = float(self._parse_stat_file()['create_time']) # According to documentation, starttime is in field 21 and the # unit is jiffies (clock ticks). # We first divide it for clock ticks and then add uptime returning # seconds since the epoch, in UTC. # Also use cached value if available. bt = BOOT_TIME or boot_time() - return (float(values[20]) / CLOCK_TICKS) + bt + return (ctime / CLOCK_TICKS) + bt @wrap_exceptions def memory_info(self): @@ -2013,7 +2026,7 @@ def rlimit(self, resource, limits=None): @wrap_exceptions def status(self): - letter = self._parse_stat_file()[1] + letter = self._parse_stat_file()['status'] if PY3: letter = letter.decode() # XXX is '?' legit? (we're not supposed to return it anyway) @@ -2082,7 +2095,7 @@ def num_fds(self): @wrap_exceptions def ppid(self): - return int(self._parse_stat_file()[2]) + return int(self._parse_stat_file()['ppid']) @wrap_exceptions def uids(self, _uids_re=re.compile(br'Uid:\t(\d+)\t(\d+)\t(\d+)')): From 8351fa4ff642d997cd478a7d216b661e62a5f696 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 11 Dec 2018 22:05:40 +0100 Subject: [PATCH 175/182] use PROCESS_QUERY_LIMITED_INFORMATION also for username() --- psutil/_psutil_windows.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/psutil/_psutil_windows.c b/psutil/_psutil_windows.c index cbd7ea8cb..ce44258a3 100644 --- a/psutil/_psutil_windows.c +++ b/psutil/_psutil_windows.c @@ -1428,7 +1428,8 @@ psutil_proc_username(PyObject *self, PyObject *args) { if (! PyArg_ParseTuple(args, "l", &pid)) return NULL; - processHandle = psutil_handle_from_pid(pid, PROCESS_QUERY_INFORMATION); + processHandle = psutil_handle_from_pid( + pid, PROCESS_QUERY_LIMITED_INFORMATION); if (processHandle == NULL) return NULL; From 2cdf81db322822ba8fb23ed67523aacb6539da95 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 13 Dec 2018 15:54:37 +0100 Subject: [PATCH 176/182] #1373: different approach to oneshot() cache (pass Process instances around - which is faster) --- psutil/__init__.py | 16 ++++++++-------- psutil/_common.py | 25 +++++++++++++------------ psutil/_psaix.py | 14 +++++++------- psutil/_psbsd.py | 6 +++--- psutil/_pslinux.py | 14 +++++++------- psutil/_psosx.py | 10 +++++----- psutil/_pssunos.py | 14 +++++++------- psutil/_pswindows.py | 6 +++--- psutil/tests/test_misc.py | 4 ++-- psutil/tests/test_process.py | 15 +++++++++++++++ 10 files changed, 70 insertions(+), 54 deletions(-) diff --git a/psutil/__init__.py b/psutil/__init__.py index 5a5720d7a..78ff985df 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -471,23 +471,23 @@ def oneshot(self): self._oneshot_inctx = True try: # cached in case cpu_percent() is used - self.cpu_times.cache_activate() + self.cpu_times.cache_activate(self) # cached in case memory_percent() is used - self.memory_info.cache_activate() + self.memory_info.cache_activate(self) # cached in case parent() is used - self.ppid.cache_activate() + self.ppid.cache_activate(self) # cached in case username() is used if POSIX: - self.uids.cache_activate() + self.uids.cache_activate(self) # specific implementation cache self._proc.oneshot_enter() yield finally: - self.cpu_times.cache_deactivate() - self.memory_info.cache_deactivate() - self.ppid.cache_deactivate() + self.cpu_times.cache_deactivate(self) + self.memory_info.cache_deactivate(self) + self.ppid.cache_deactivate(self) if POSIX: - self.uids.cache_deactivate() + self.uids.cache_deactivate(self) self._proc.oneshot_exit() self._oneshot_inctx = False diff --git a/psutil/_common.py b/psutil/_common.py index bee957927..f498bb903 100644 --- a/psutil/_common.py +++ b/psutil/_common.py @@ -327,7 +327,7 @@ def memoize_when_activated(fun): 1 >>> >>> # activated - >>> foo.cache_activate() + >>> foo.cache_activate(self) >>> foo() 1 >>> foo() @@ -336,26 +336,27 @@ def memoize_when_activated(fun): """ @functools.wraps(fun) def wrapper(self): - if not wrapper.cache_activated: + if not hasattr(self, "_cache"): return fun(self) else: try: - ret = cache[fun] + ret = self._cache[fun] except KeyError: - ret = cache[fun] = fun(self) + ret = self._cache[fun] = fun(self) return ret - def cache_activate(): - """Activate cache.""" - wrapper.cache_activated = True + def cache_activate(proc): + """Activate cache. Expects a Process instance. Cache will be + stored as a "_cache" instance attribute.""" + proc._cache = {} - def cache_deactivate(): + def cache_deactivate(proc): """Deactivate and clear cache.""" - wrapper.cache_activated = False - cache.clear() + try: + del proc._cache + except AttributeError: + pass - cache = {} - wrapper.cache_activated = False wrapper.cache_activate = cache_activate wrapper.cache_deactivate = cache_deactivate return wrapper diff --git a/psutil/_psaix.py b/psutil/_psaix.py index 7ba212dbf..9975545aa 100644 --- a/psutil/_psaix.py +++ b/psutil/_psaix.py @@ -354,7 +354,7 @@ def wrapper(self, *args, **kwargs): class Process(object): """Wrapper class around underlying C implementation.""" - __slots__ = ["pid", "_name", "_ppid", "_procfs_path"] + __slots__ = ["pid", "_name", "_ppid", "_procfs_path", "_cache"] def __init__(self, pid): self.pid = pid @@ -363,14 +363,14 @@ def __init__(self, pid): self._procfs_path = get_procfs_path() def oneshot_enter(self): - self._proc_name_and_args.cache_activate() - self._proc_basic_info.cache_activate() - self._proc_cred.cache_activate() + self._proc_name_and_args.cache_activate(self) + self._proc_basic_info.cache_activate(self) + self._proc_cred.cache_activate(self) def oneshot_exit(self): - self._proc_name_and_args.cache_deactivate() - self._proc_basic_info.cache_deactivate() - self._proc_cred.cache_deactivate() + self._proc_name_and_args.cache_deactivate(self) + self._proc_basic_info.cache_deactivate(self) + self._proc_cred.cache_deactivate(self) @memoize_when_activated def _proc_name_and_args(self): diff --git a/psutil/_psbsd.py b/psutil/_psbsd.py index 1dc723128..6683a2005 100644 --- a/psutil/_psbsd.py +++ b/psutil/_psbsd.py @@ -594,7 +594,7 @@ def wrap_exceptions_procfs(inst): class Process(object): """Wrapper class around underlying C implementation.""" - __slots__ = ["pid", "_name", "_ppid"] + __slots__ = ["pid", "_name", "_ppid", "_cache"] def __init__(self, pid): self.pid = pid @@ -609,10 +609,10 @@ def oneshot(self): return ret def oneshot_enter(self): - self.oneshot.cache_activate() + self.oneshot.cache_activate(self) def oneshot_exit(self): - self.oneshot.cache_deactivate() + self.oneshot.cache_deactivate(self) @wrap_exceptions def name(self): diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index 880be2c82..5c8cc20c4 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -1528,7 +1528,7 @@ def wrapper(self, *args, **kwargs): class Process(object): """Linux process implementation.""" - __slots__ = ["pid", "_name", "_ppid", "_procfs_path"] + __slots__ = ["pid", "_name", "_ppid", "_procfs_path", "_cache"] def __init__(self, pid): self.pid = pid @@ -1585,14 +1585,14 @@ def _read_smaps_file(self): return f.read().strip() def oneshot_enter(self): - self._parse_stat_file.cache_activate() - self._read_status_file.cache_activate() - self._read_smaps_file.cache_activate() + self._parse_stat_file.cache_activate(self) + self._read_status_file.cache_activate(self) + self._read_smaps_file.cache_activate(self) def oneshot_exit(self): - self._parse_stat_file.cache_deactivate() - self._read_status_file.cache_deactivate() - self._read_smaps_file.cache_deactivate() + self._parse_stat_file.cache_deactivate(self) + self._read_status_file.cache_deactivate(self) + self._read_smaps_file.cache_deactivate(self) @wrap_exceptions def name(self): diff --git a/psutil/_psosx.py b/psutil/_psosx.py index 94e22bc72..015c5b411 100644 --- a/psutil/_psosx.py +++ b/psutil/_psosx.py @@ -380,7 +380,7 @@ def catch_zombie(proc): class Process(object): """Wrapper class around underlying C implementation.""" - __slots__ = ["pid", "_name", "_ppid"] + __slots__ = ["pid", "_name", "_ppid", "_cache"] def __init__(self, pid): self.pid = pid @@ -403,12 +403,12 @@ def _get_pidtaskinfo(self): return ret def oneshot_enter(self): - self._get_kinfo_proc.cache_activate() - self._get_pidtaskinfo.cache_activate() + self._get_kinfo_proc.cache_activate(self) + self._get_pidtaskinfo.cache_activate(self) def oneshot_exit(self): - self._get_kinfo_proc.cache_deactivate() - self._get_pidtaskinfo.cache_deactivate() + self._get_kinfo_proc.cache_deactivate(self) + self._get_pidtaskinfo.cache_deactivate(self) @wrap_exceptions def name(self): diff --git a/psutil/_pssunos.py b/psutil/_pssunos.py index e2f33a3ae..730af3930 100644 --- a/psutil/_pssunos.py +++ b/psutil/_pssunos.py @@ -368,7 +368,7 @@ def wrapper(self, *args, **kwargs): class Process(object): """Wrapper class around underlying C implementation.""" - __slots__ = ["pid", "_name", "_ppid", "_procfs_path"] + __slots__ = ["pid", "_name", "_ppid", "_procfs_path", "_cache"] def __init__(self, pid): self.pid = pid @@ -377,14 +377,14 @@ def __init__(self, pid): self._procfs_path = get_procfs_path() def oneshot_enter(self): - self._proc_name_and_args.cache_activate() - self._proc_basic_info.cache_activate() - self._proc_cred.cache_activate() + self._proc_name_and_args.cache_activate(self) + self._proc_basic_info.cache_activate(self) + self._proc_cred.cache_activate(self) def oneshot_exit(self): - self._proc_name_and_args.cache_deactivate() - self._proc_basic_info.cache_deactivate() - self._proc_cred.cache_deactivate() + self._proc_name_and_args.cache_deactivate(self) + self._proc_basic_info.cache_deactivate(self) + self._proc_cred.cache_deactivate(self) @memoize_when_activated def _proc_name_and_args(self): diff --git a/psutil/_pswindows.py b/psutil/_pswindows.py index 2bc9c9dd0..bb5882422 100644 --- a/psutil/_pswindows.py +++ b/psutil/_pswindows.py @@ -645,7 +645,7 @@ def wrapper(self, *args, **kwargs): class Process(object): """Wrapper class around underlying C implementation.""" - __slots__ = ["pid", "_name", "_ppid"] + __slots__ = ["pid", "_name", "_ppid", "_cache"] def __init__(self, pid): self.pid = pid @@ -655,10 +655,10 @@ def __init__(self, pid): # --- oneshot() stuff def oneshot_enter(self): - self.oneshot_info.cache_activate() + self.oneshot_info.cache_activate(self) def oneshot_exit(self): - self.oneshot_info.cache_deactivate() + self.oneshot_info.cache_deactivate(self) @memoize_when_activated def oneshot_info(self): diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index 3056abc0a..93132b556 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -261,14 +261,14 @@ def foo(self): # activate calls = [] - f.foo.cache_activate() + f.foo.cache_activate(f) f.foo() f.foo() self.assertEqual(len(calls), 1) # deactivate calls = [] - f.foo.cache_deactivate() + f.foo.cache_deactivate(f) f.foo() f.foo() self.assertEqual(len(calls), 2) diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index 2126e32ad..cd72be850 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -1195,6 +1195,21 @@ def test_oneshot_twice(self): p.cpu_times() self.assertEqual(m.call_count, 2) + def test_oneshot_cache(self): + # Make sure oneshot() cache is nonglobal. Instead it's + # supposed to be bound to the Process instance, see: + # https://github.com/giampaolo/psutil/issues/1373 + p1, p2 = create_proc_children_pair() + p1_ppid = p1.ppid() + p2_ppid = p2.ppid() + self.assertNotEqual(p1_ppid, p2_ppid) + with p1.oneshot(): + self.assertEqual(p1.ppid(), p1_ppid) + self.assertEqual(p2.ppid(), p2_ppid) + with p2.oneshot(): + self.assertEqual(p1.ppid(), p1_ppid) + self.assertEqual(p2.ppid(), p2_ppid) + def test_halfway_terminated_process(self): # Test that NoSuchProcess exception gets raised in case the # process dies after we create the Process object. From 495bb454d12f72f00ea86a5cad196425e3ae2a78 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 13 Dec 2018 16:27:44 +0100 Subject: [PATCH 177/182] pdate HISTORY --- HISTORY.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index 59bf9f678..50c839068 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -20,6 +20,8 @@ XXXX-XX-XX EccoTheFlintstone) - 1370_: [Windows] improper usage of CloseHandle() may lead to override the original error code when raising an exception. +- 1373_: incorrect handling of cache in Process.oneshot() context causes + Process instances to return incorrect results. - 1376_: [Windows] OpenProcess() now uses PROCESS_QUERY_LIMITED_INFORMATION access rights wherever possible, resulting in less AccessDenied exceptions being thrown for system processes. From b3b5d4293c60bc4390a5a5a39491e985626c9139 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 13 Dec 2018 16:48:57 +0100 Subject: [PATCH 178/182] fix #1111: use a lock to make Process.oneshot() thread safe --- HISTORY.rst | 1 + psutil/__init__.py | 74 +++++++++++++++++++++++++--------------------- psutil/_common.py | 17 ++++++----- 3 files changed, 51 insertions(+), 41 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 50c839068..d704e39d0 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -13,6 +13,7 @@ XXXX-XX-XX **Bug fixes** +- 1111_: Process.oneshot() is now thread safe. - 1354_: [Linux] disk_io_counters() fails on Linux kernel 4.18+. - 1357_: [Linux] Process' memory_maps() and io_counters() method are no longer exposed if not supported by the kernel. diff --git a/psutil/__init__.py b/psutil/__init__.py index 78ff985df..a0258b209 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -31,6 +31,7 @@ import signal import subprocess import sys +import threading import time try: import pwd @@ -352,7 +353,7 @@ def _init(self, pid, _ignore_nsp=False): self._create_time = None self._gone = False self._hash = None - self._oneshot_inctx = False + self._lock = threading.RLock() # used for caching on Windows only (on POSIX ppid may change) self._ppid = None # platform-specific modules define an _psplatform.Process @@ -456,40 +457,45 @@ def oneshot(self): ... >>> """ - if self._oneshot_inctx: - # NOOP: this covers the use case where the user enters the - # context twice. Since as_dict() internally uses oneshot() - # I expect that the code below will be a pretty common - # "mistake" that the user will make, so let's guard - # against that: - # - # >>> with p.oneshot(): - # ... p.as_dict() - # ... - yield - else: - self._oneshot_inctx = True - try: - # cached in case cpu_percent() is used - self.cpu_times.cache_activate(self) - # cached in case memory_percent() is used - self.memory_info.cache_activate(self) - # cached in case parent() is used - self.ppid.cache_activate(self) - # cached in case username() is used - if POSIX: - self.uids.cache_activate(self) - # specific implementation cache - self._proc.oneshot_enter() + with self._lock: + if hasattr(self, "_cache"): + # NOOP: this covers the use case where the user enters the + # context twice: + # + # >>> with p.oneshot(): + # ... with p.oneshot(): + # ... + # + # Also, since as_dict() internally uses oneshot() + # I expect that the code below will be a pretty common + # "mistake" that the user will make, so let's guard + # against that: + # + # >>> with p.oneshot(): + # ... p.as_dict() + # ... yield - finally: - self.cpu_times.cache_deactivate(self) - self.memory_info.cache_deactivate(self) - self.ppid.cache_deactivate(self) - if POSIX: - self.uids.cache_deactivate(self) - self._proc.oneshot_exit() - self._oneshot_inctx = False + else: + try: + # cached in case cpu_percent() is used + self.cpu_times.cache_activate(self) + # cached in case memory_percent() is used + self.memory_info.cache_activate(self) + # cached in case parent() is used + self.ppid.cache_activate(self) + # cached in case username() is used + if POSIX: + self.uids.cache_activate(self) + # specific implementation cache + self._proc.oneshot_enter() + yield + finally: + self.cpu_times.cache_deactivate(self) + self.memory_info.cache_deactivate(self) + self.ppid.cache_deactivate(self) + if POSIX: + self.uids.cache_deactivate(self) + self._proc.oneshot_exit() def as_dict(self, attrs=None, ad_value=None): """Utility method returning process information as a diff --git a/psutil/_common.py b/psutil/_common.py index f498bb903..b809a79f6 100644 --- a/psutil/_common.py +++ b/psutil/_common.py @@ -336,14 +336,17 @@ def memoize_when_activated(fun): """ @functools.wraps(fun) def wrapper(self): - if not hasattr(self, "_cache"): + try: + # case 1: we previously entered oneshot() ctx + ret = self._cache[fun] + except AttributeError: + # case 2: we never entered oneshot() ctx return fun(self) - else: - try: - ret = self._cache[fun] - except KeyError: - ret = self._cache[fun] = fun(self) - return ret + except KeyError: + # case 3: we entered oneshot() ctx but there's no cache + # for this entry yet + ret = self._cache[fun] = fun(self) + return ret def cache_activate(proc): """Activate cache. Expects a Process instance. Cache will be From 4ae4f944c1311f471d8506a92eab8ac52401d623 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 25 Dec 2018 13:02:20 +0100 Subject: [PATCH 179/182] update readme --- IDEAS | 4 ++++ README.rst | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/IDEAS b/IDEAS index babc0e8be..046044a30 100644 --- a/IDEAS +++ b/IDEAS @@ -19,6 +19,8 @@ PLATFORMS FEATURES ======== +- set process name/title + - #1115: users() idle time. - #1102: Process.is64bit(). @@ -151,6 +153,8 @@ FEATURES - #550: number of threads per core. +- cpu_percent() and cpu_times_percent() use global vars so are not thread safe. + BUGFIXES ======== diff --git a/README.rst b/README.rst index 2fa59be65..01fdf00bb 100644 --- a/README.rst +++ b/README.rst @@ -1,5 +1,5 @@ -.. image:: http://pepy.tech/badge/psutil - :target: http://pepy.tech/project/psutil +.. image:: https://pepy.tech/badge/psutil/month + :target: https://pepy.tech/project/psutil :alt: Downloads .. image:: https://img.shields.io/travis/giampaolo/psutil/master.svg?maxAge=3600&label=Linux%20/%20macOS From 1b2148b7cbc0f7961a2f1f871371c754348f34db Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 23 Jan 2019 19:06:04 +0100 Subject: [PATCH 180/182] fix win num_handles() test --- psutil/tests/test_windows.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/psutil/tests/test_windows.py b/psutil/tests/test_windows.py index ffa763d09..4633f7596 100755 --- a/psutil/tests/test_windows.py +++ b/psutil/tests/test_windows.py @@ -518,15 +518,15 @@ def test_num_handles(self): import ctypes.wintypes PROCESS_QUERY_INFORMATION = 0x400 handle = ctypes.windll.kernel32.OpenProcess( - PROCESS_QUERY_INFORMATION, 0, os.getpid()) + PROCESS_QUERY_INFORMATION, 0, self.pid) self.addCleanup(ctypes.windll.kernel32.CloseHandle, handle) + hndcnt = ctypes.wintypes.DWORD() ctypes.windll.kernel32.GetProcessHandleCount( handle, ctypes.byref(hndcnt)) sys_value = hndcnt.value - psutil_value = psutil.Process().num_handles() - ctypes.windll.kernel32.CloseHandle(handle) - self.assertEqual(psutil_value, sys_value + 1) + psutil_value = psutil.Process(self.pid).num_handles() + self.assertEqual(psutil_value, sys_value) @unittest.skipIf(not WINDOWS, "WINDOWS only") From c6b3e929deb182d4db6007548571ad8ddb32bd87 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 23 Jan 2019 19:23:18 +0100 Subject: [PATCH 181/182] pre-release --- HISTORY.rst | 2 +- docs/index.rst | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index d704e39d0..44afa960f 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -3,7 +3,7 @@ 5.5.0 ===== -XXXX-XX-XX +2019-0-23 **Enhancements** diff --git a/docs/index.rst b/docs/index.rst index 85506a280..643bb571f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -2626,6 +2626,10 @@ take a look at the Timeline ======== +- 2019-01-23: + `5.5.0 `__ - + `what's new `__ - + `diff `__ - 2018-10-30: `5.4.8 `__ - `what's new `__ - From 6f4a6228998df48ee09413377785d13d2eec7998 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 25 Jan 2019 16:36:17 +0100 Subject: [PATCH 182/182] #1394 / windows / process exe(): convert errno 0 into ERROR_ACCESS_DENIED; errno 0 occurs when the Python process runs in 'Virtual Secure Mode' --- HISTORY.rst | 10 ++++++++++ psutil/_psutil_linux.c | 5 ++++- psutil/_psutil_windows.c | 6 +++++- psutil/_pswindows.py | 4 +++- 4 files changed, 22 insertions(+), 3 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 44afa960f..095c90623 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,5 +1,15 @@ *Bug tracker at https://github.com/giampaolo/psutil/issues* +5.5.1 +===== + +XXXX-XX-XX + +**Bug fixes** + +- 1394_: [Windows] Process.exe() returns "[Error 0] The operation completed + successfully" when Python process runs in "Virtual Secure Mode". + 5.5.0 ===== diff --git a/psutil/_psutil_linux.c b/psutil/_psutil_linux.c index bd27b5f9c..5b7a56ad9 100644 --- a/psutil/_psutil_linux.c +++ b/psutil/_psutil_linux.c @@ -211,6 +211,7 @@ psutil_disk_partitions(PyObject *self, PyObject *args) { file = setmntent(mtab_path, "r"); Py_END_ALLOW_THREADS if ((file == 0) || (file == NULL)) { + psutil_debug("setmntent() failed"); PyErr_SetFromErrnoWithFilename(PyExc_OSError, mtab_path); goto error; } @@ -298,8 +299,10 @@ psutil_proc_cpu_affinity_get(PyObject *self, PyObject *args) { while (1) { setsize = CPU_ALLOC_SIZE(ncpus); mask = CPU_ALLOC(ncpus); - if (mask == NULL) + if (mask == NULL) { + psutil_debug("CPU_ALLOC() failed"); return PyErr_NoMemory(); + } if (sched_getaffinity(pid, setsize, mask) == 0) break; CPU_FREE(mask); diff --git a/psutil/_psutil_windows.c b/psutil/_psutil_windows.c index ce44258a3..4251e0c71 100644 --- a/psutil/_psutil_windows.c +++ b/psutil/_psutil_windows.c @@ -765,7 +765,11 @@ psutil_proc_exe(PyObject *self, PyObject *args) { if (NULL == hProcess) return NULL; if (GetProcessImageFileNameW(hProcess, exe, MAX_PATH) == 0) { - PyErr_SetFromWindowsErr(0); + // https://github.com/giampaolo/psutil/issues/1394 + if (GetLastError() == 0) + PyErr_SetFromWindowsErr(ERROR_ACCESS_DENIED); + else + PyErr_SetFromWindowsErr(0); CloseHandle(hProcess); return NULL; } diff --git a/psutil/_pswindows.py b/psutil/_pswindows.py index bb5882422..664d5b6b1 100644 --- a/psutil/_pswindows.py +++ b/psutil/_pswindows.py @@ -698,7 +698,9 @@ def exe(self): # see https://github.com/giampaolo/psutil/issues/528 if self.pid in (0, 4): raise AccessDenied(self.pid, self._name) - return py2_strencode(convert_dos_path(cext.proc_exe(self.pid))) + exe = cext.proc_exe(self.pid) + exe = convert_dos_path(exe) + return py2_strencode(exe) @wrap_exceptions def cmdline(self):